diff --git a/.gitignore b/.gitignore index 9359a483c4b..fafcdc86c4f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,8 @@ __pycache__/ __version__.py .glv settings.xml -tools/ +tools/* +!tools/tinkerpop-docs/ [Dd]ebug/ [Rr]elease/ [Oo]bj/ diff --git a/bin/process-docs.sh b/bin/process-docs.sh index 7236faad887..10bd27c8a41 100755 --- a/bin/process-docs.sh +++ b/bin/process-docs.sh @@ -18,96 +18,268 @@ # under the License. # -pushd "$(dirname $0)/.." > /dev/null - -NOCLEAN= - -DRYRUN= -DRYRUN_DOCS= -FULLRUN_DOCS= - -makeAbsPaths () { - for doc in $(tr ',' $'\n' <<< "$1"); do - if [ -d $doc ]; then - for d in $(find "$doc" -name "*.asciidoc"); do - echo $(cd $(dirname "$d") && pwd -P)/$(basename "$d") - done - else - echo $(cd $(dirname "$doc") && pwd -P)/$(basename "$doc") - fi - done | paste -sd ',' - -} +# Orchestration script for the TinkerPop documentation build. +# +# Full build (--fullRun, default): +# Validates console/server distributions, installs plugins, starts +# Gremlin Server and Gephi mock, then invokes Maven with the +# AsciidoctorJ extension to render docs with live code execution. +# +# Dry-run (--dryRun): +# Invokes Maven with -Dgremlin-docs-dryrun=true. No server, console, +# or plugins required. + +set -e + +cd "$(dirname "$0")/.." +TP_HOME="$(pwd)" -while [[ $# -gt 0 ]] -do - key="$1" - case $key in - -n|--noClean) - NOCLEAN=1 - shift - ;; - -d|--dryRun) - DRYRUN=1 - shift - if [[ $# -gt 0 ]] && [[ $1 != -* ]]; then - DRYRUN_DOCS=$(makeAbsPaths "$1") - shift - else - DRYRUN_DOCS="*" - fi - ;; - -f|--fullRun) - DRYRUN=1 - DRYRUN_DOCS=${DRYRUN_DOCS:-"*"} - shift - FULLRUN_DOCS=$(makeAbsPaths "$1") - shift - ;; - *) - # unknown option - shift - ;; +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +MODE="full" +NOCLEAN="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --fullRun|-f) MODE="full"; shift ;; + --dryRun|-d) MODE="dry"; shift ;; + --noClean|-n) NOCLEAN="1"; shift ;; + *) echo "Unknown option: $1"; exit 1 ;; esac done -if [ -z ${NOCLEAN} ]; then - rm -rf ~/.groovy/grapes/org.apache.tinkerpop/ - if hash hadoop 2> /dev/null; then - hadoop fs -rm -r "hadoop-gremlin-*-libs" > /dev/null 2>&1 +# --------------------------------------------------------------------------- +# Cleanup trap +# --------------------------------------------------------------------------- +GREMLIN_SERVER_PID="" +GEPHI_MOCK_PID="" + +cleanup() { + set +e + [ -n "${GREMLIN_SERVER_PID}" ] && kill "${GREMLIN_SERVER_PID}" 2>/dev/null + [ -n "${GEPHI_MOCK_PID}" ] && kill "${GEPHI_MOCK_PID}" 2>/dev/null +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Dry-run mode +# --------------------------------------------------------------------------- +if [ "${MODE}" == "dry" ]; then + if [ -z "${NOCLEAN}" ]; then + rm -rf target/postprocess-asciidoc target/doc-source target/docs 2>/dev/null || true fi + echo "Copying docs sources to target/postprocess-asciidoc/..." + mkdir -p target/postprocess-asciidoc + cp -r docs/{static,stylesheets} target/postprocess-asciidoc/ + cp -r docs/src/* target/postprocess-asciidoc/ + mvn process-resources -pl . -Dasciidoc -Dgremlin.docs.dryrun=true + exit $? fi -if [ ${DRYRUN} ] && [ "${DRYRUN_DOCS}" == "*" ] && [ -z "${FULLRUN_DOCS}" ]; then +# --------------------------------------------------------------------------- +# Full build mode +# --------------------------------------------------------------------------- - mkdir -p target/postprocess-asciidoc/tmp - cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - cp -R docs/src/. target/postprocess-asciidoc/ - ec=$? +# Resolve version from pom.xml +TP_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout 2>/dev/null || \ + grep -A1 'tinkerpop' pom.xml | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) -else +# 1. Validate console distribution +CONSOLE_DIR=$(ls -d gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone 2>/dev/null | head -n1) +if [ -z "${CONSOLE_DIR}" ] || [ ! -d "${CONSOLE_DIR}" ]; then + echo "ERROR: Gremlin Console distribution not found." + echo "Build it first: mvn clean install -pl :gremlin-console -am -DskipTests" + exit 1 +fi +CONSOLE_HOME="$(cd "${CONSOLE_DIR}" && pwd)" + +# 2. Validate server distribution +SERVER_DIR=$(ls -d gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone 2>/dev/null | head -n1) +if [ -z "${SERVER_DIR}" ] || [ ! -d "${SERVER_DIR}" ]; then + echo "ERROR: Gremlin Server distribution not found." + echo "Build it first: mvn clean install -pl :gremlin-server -am -DskipTests" + exit 1 +fi +SERVER_HOME="$(cd "${SERVER_DIR}" && pwd)" - GEPHI_MOCK= +# Clear any plugin directories parked aside by a previous (possibly interrupted) run. The +# extension moves ext/ to ext-disabled/ when excluding a plugin per-book; a +# leftover ext-disabled/ from a crashed build would otherwise shadow the freshly-installed +# plugins and break the next run's console restarts. +rm -rf "${CONSOLE_HOME}/ext-disabled" - trap cleanup EXIT +# 3. Install plugins into console +echo "Installing plugins into console..." - function cleanup() { - [ ${GEPHI_MOCK} ] && kill ${GEPHI_MOCK} - } +# Copy a plugin's dependency jars onto the console classpath via ext//plugin/ (which +# bin/gremlin.sh globs) rather than the shared lib/. This keeps each plugin's transitive deps +# isolatable so the docs extension can exclude conflicting plugins per-book (e.g. Neo4j's +# Scala 2.11 vs Spark's 2.12) by moving the plugin directory off the classpath. Jars already +# present in lib/ (core gremlin deps) and slf4j/logback-classic are skipped to avoid duplicate +# classpath entries and logger bindings -- mirroring the console's own :install (DependencyGrabber). +copy_deps_to_plugin() { + local src_dir="$1" plugin="$2" + local plugin_dir="${CONSOLE_HOME}/ext/${plugin}/plugin" + mkdir -p "${plugin_dir}" + local jar base + for jar in "${src_dir}"/*.jar; do + [ -e "${jar}" ] || continue + base=$(basename "${jar}") + case "${base}" in slf4j-*|logback-classic-*) continue ;; esac + [ -e "${CONSOLE_HOME}/lib/${base}" ] && continue + cp "${jar}" "${plugin_dir}/" 2>/dev/null + done +} - nc -z localhost 8080 || ( - bin/gephi-mock.py > /dev/null 2>&1 & - GEPHI_MOCK=$! - ) +PLUGINS="hadoop-gremlin spark-gremlin neo4j-gremlin sparql-gremlin" +for plugin in ${PLUGINS}; do + PLUGIN_DIR="${plugin}/target/${plugin}-${TP_VERSION}-standalone" + if [ -d "${PLUGIN_DIR}" ]; then + echo " * installing ${plugin} (standalone)" + cp -r "${PLUGIN_DIR}" "${CONSOLE_HOME}/ext/${plugin}" + mkdir -p "${CONSOLE_HOME}/ext/${plugin}/plugin" + cp "${plugin}/target/${plugin}-${TP_VERSION}.jar" "${CONSOLE_HOME}/ext/${plugin}/plugin/" 2>/dev/null + copy_deps_to_plugin "${CONSOLE_HOME}/ext/${plugin}/lib" "${plugin}" + elif [ -f "${plugin}/target/${plugin}-${TP_VERSION}.jar" ]; then + echo " * installing ${plugin} (jar + dependencies)" + mkdir -p "${CONSOLE_HOME}/ext/${plugin}/lib" + mkdir -p "${CONSOLE_HOME}/ext/${plugin}/plugin" + cp "${plugin}/target/${plugin}-${TP_VERSION}.jar" "${CONSOLE_HOME}/ext/${plugin}/lib/" + cp "${plugin}/target/${plugin}-${TP_VERSION}.jar" "${CONSOLE_HOME}/ext/${plugin}/plugin/" + cp "${plugin}"/target/dependency/*.jar "${CONSOLE_HOME}/ext/${plugin}/lib/" 2>/dev/null || \ + mvn dependency:copy-dependencies -pl "${plugin}" -DoutputDirectory="${CONSOLE_HOME}/ext/${plugin}/lib" -q + copy_deps_to_plugin "${CONSOLE_HOME}/ext/${plugin}/lib" "${plugin}" + else + echo " * WARNING: ${plugin} not found" + fi +done - docs/preprocessor/preprocess.sh "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - ec=$? +# 3b. Resolve the Neo4j database implementation onto the console classpath. +# neo4j-gremlin declares neo4j-tinkerpop-api-impl as a Gremlin-Plugin-Dependencies manifest +# entry (test-scoped / gated behind the includeNeo4j profile), so it is not bundled by the +# local jar copy above. The old console ':install' flow fetched it via DependencyGrabber; +# here we resolve it (and its transitive deps) from Maven so Neo4jGraph blocks can execute. +NEO4J_IMPL_VERSION="0.9-3.4.0" +NEO4J_PLUGIN_LIB="${CONSOLE_HOME}/ext/neo4j-gremlin/lib" +if [ -d "${NEO4J_PLUGIN_LIB}" ] && ! ls "${NEO4J_PLUGIN_LIB}"/neo4j-tinkerpop-api-impl-*.jar >/dev/null 2>&1; then + echo " * resolving Neo4j implementation (neo4j-tinkerpop-api-impl:${NEO4J_IMPL_VERSION})" + NEO4J_POM="${TP_HOME}/target/neo4j-impl-pom.xml" + cat > "${NEO4J_POM}" <4.0.0 +org.apache.tinkerpop.docsneo4j-impl-resolver1 +org.neo4jneo4j-tinkerpop-api-impl${NEO4J_IMPL_VERSION} +POM + mvn -q -f "${NEO4J_POM}" dependency:copy-dependencies -DoutputDirectory="${NEO4J_PLUGIN_LIB}" + # Drop ONLY the conflicting io.netty 4.x jar that Neo4j pulls in (netty-all-4.1.24): it + # contains an older io.netty.handler.codec.http.websocketx.WebSocketClientHandshaker13 that + # shadows the console driver's 4.1.125 class and breaks ':remote' server connections with a + # NoSuchMethodError. Keep netty-3.9.x (org.jboss.netty package) -- it does NOT conflict and + # is required by Neo4j 3.4's IO layer. + rm -f "${NEO4J_PLUGIN_LIB}"/netty-all-4.*.jar + copy_deps_to_plugin "${NEO4J_PLUGIN_LIB}" "neo4j-gremlin" fi -if [ $ec -eq 0 ]; then - mvn process-resources -Dasciidoc && docs/postprocessor/postprocess.sh - ec=$? +# 4. Register plugins in console +echo "Registering plugins..." +# Write plugins.txt deterministically rather than appending to whatever state a prior run left: +# the console rewrites this file to the set of successfully-activated plugins on shutdown, so a +# previous (possibly failed) run can leave it missing TinkerGraph/Credentials, which would fail +# the first doc block with "No such property: TinkerFactory". Lightweight built-in plugins are +# listed before the heavy graph plugins so activation order is stable. +cat > "${CONSOLE_HOME}/ext/plugins.txt" <<'EOF' +org.apache.tinkerpop.gremlin.console.jsr223.DriverGremlinPlugin +org.apache.tinkerpop.gremlin.console.jsr223.UtilitiesGremlinPlugin +org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin +org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.CredentialGraphGremlinPlugin +org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopGremlinPlugin +org.apache.tinkerpop.gremlin.spark.jsr223.SparkGremlinPlugin +org.apache.tinkerpop.gremlin.neo4j.jsr223.Neo4jGremlinPlugin +org.apache.tinkerpop.gremlin.sparql.jsr223.SparqlGremlinPlugin +EOF + +# 5. Copy hadoop config to console classpath +HADOOP_CONF_SRC="tools/tinkerpop-docs/src/main/resources/hadoop-conf" +cp "${HADOOP_CONF_SRC}/core-site.xml" "${CONSOLE_HOME}/conf/" +cp "${HADOOP_CONF_SRC}/hadoop-docs.properties" "${CONSOLE_HOME}/conf/" + +# 5. Start Gremlin Server +echo "Starting Gremlin Server..." +mkdir -p target + +# Fail fast if port 8182 is already in use. The docs ':remote connect' blocks +# target localhost:8182, so any other service on that port (a stale Gremlin +# Server, or an unrelated process that happens to claim 8182) will cause our +# server to fail binding while the readiness check still passes -- the console +# then connects to the wrong service and WebSocket handshakes fail, dumping +# large stacktraces into the rendered docs. +if nc -z localhost 8182 2>/dev/null; then + echo "ERROR: Port 8182 is already in use by another process." + echo " Gremlin Server needs this port for the docs ':remote' examples." + echo " Identify the process with 'lsof -nP -iTCP:8182' and stop it," + echo " then re-run the docs build." + exit 1 fi +pushd "${SERVER_HOME}" > /dev/null +bin/gremlin-server.sh conf/gremlin-server-modern.yaml > "${TP_HOME}/target/gremlin-server-docs.log" 2>&1 & +GREMLIN_SERVER_PID=$! popd > /dev/null +# Wait for server to be ready (port 8182) +echo -n "Waiting for Gremlin Server on port 8182..." +for i in $(seq 1 30); do + # Detect early server failure (e.g. bind error) so we don't wait the full timeout + if ! kill -0 "${GREMLIN_SERVER_PID}" 2>/dev/null; then + echo " FAILED" + echo "ERROR: Gremlin Server process exited during startup. See target/gremlin-server-docs.log" + exit 1 + fi + if nc -z localhost 8182 2>/dev/null; then + echo " ready." + break + fi + if [ $i -eq 30 ]; then + echo " TIMEOUT" + echo "ERROR: Gremlin Server failed to start within 30 seconds." + exit 1 + fi + sleep 1 + echo -n "." +done + +# 6. Start Gephi mock (port 8080) +if ! nc -z localhost 8080 2>/dev/null; then + echo "Starting Gephi mock on port 8080..." + bin/gephi-mock.py > /dev/null 2>&1 & + GEPHI_MOCK_PID=$! +fi + +# 7. Resolve HADOOP_GREMLIN_LIBS path +HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/hadoop-gremlin/lib" + +# 8. Copy source docs to staging area (replaces old preprocessor copy) +echo "Copying docs sources to target/postprocess-asciidoc/..." +mkdir -p target/postprocess-asciidoc +cp -r docs/{static,stylesheets} target/postprocess-asciidoc/ +cp -r docs/src/* target/postprocess-asciidoc/ + +# 9. Invoke Maven with AsciidoctorJ extension attributes +echo "Running documentation build..." +if [ -z "${NOCLEAN}" ]; then + rm -r target/doc-source target/docs 2>/dev/null || true +fi +set +e +mvn process-resources -pl . -Dasciidoc \ + -Dgremlin.docs.console.home="${CONSOLE_HOME}" \ + -Dgremlin.docs.hadoop.libs="${HADOOP_GREMLIN_LIBS}" +ec=$? +set -e + +if [ ${ec} -eq 0 ]; then + echo "Documentation build complete. Output: target/docs/htmlsingle/" +else + echo "ERROR: Documentation build failed." +fi + exit ${ec} diff --git a/bin/validate-distribution.sh b/bin/validate-distribution.sh index 7c5c3ee851f..1a1c1b738b6 100755 --- a/bin/validate-distribution.sh +++ b/bin/validate-distribution.sh @@ -209,7 +209,7 @@ if [ "${TYPE}" = "CONSOLE" ]; then SCRIPT_FILENAME="test.groovy" SCRIPT_PATH="${TMP_DIR}/${SCRIPT_FILENAME}" echo ${SCRIPT} > ${SCRIPT_PATH} - [[ `bin/gremlin.sh <<< ${SCRIPT} | ${SCRIPT_DIR}/../docs/preprocessor/control-characters.sh | grep '^==>' | sed -e 's/^==>//'` -eq 6 ]] || { echo "failed to evaluate sample script"; exit 1; } + [[ `bin/gremlin.sh <<< ${SCRIPT} | sed "s/\x1b\[[0-9;]*m//g" | tr -d "\r" | grep '^==>' | sed -e 's/^==>//'` -eq 6 ]] || { echo "failed to evaluate sample script"; exit 1; } [[ `bin/gremlin.sh -e ${SCRIPT_PATH}` -eq 6 ]] || { echo "failed to evaluate sample script using -e option"; exit 1; } CONSOLE_DIR=`pwd` cd ${TMP_DIR} diff --git a/docs/postprocessor/postprocess.sh b/docs/postprocessor/postprocess.sh deleted file mode 100755 index e7dca523b84..00000000000 --- a/docs/postprocessor/postprocess.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -pushd "$(dirname $0)/../.." > /dev/null - -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) - -if [ -d "target/docs" ]; then - - # redirect the GLV Tutorial to reference docs - sed -i "s///" target/docs/htmlsingle/tutorials/gremlin-language-variants/index.html - - find target/docs -name index.html | while read file ; do - awk -f "docs/postprocessor/processor.awk" "${file}" 2>/dev/null \ - | perl -0777 -pe 's/\/\*\n \*\/<\/span>//igs' \ - | sed "s/x\.y\.z/${TP_VERSION}/g" \ - > "${file}.tmp" && mv "${file}.tmp" "${file}" - done -fi - -popd > /dev/null diff --git a/docs/postprocessor/processor.awk b/docs/postprocessor/processor.awk deleted file mode 100644 index fbeadcccb44..00000000000 --- a/docs/postprocessor/processor.awk +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BEGIN { - firstMatch=1 - styled=0 -} - -/Licensed to the Apache Software Foundation/ { - isHeader=1 -} - -/<\/style>/ { - if (!styled) { - print ".invisible {color: rgba(0,0,0,0); font-size: 0;}" - styled=1 - } -} - -!// { - if (firstMatch || !isHeader) { - print gensub(/()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3", "g") - } -} - -// { - if (firstMatch || !isHeader) { - print gensub(/\/\/<\/span>[ ]*()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3\\\\<\/span>", "g") - } -} - -/under the License\./ { - firstMatch=0 - isHeader=0 -} diff --git a/docs/preprocessor/awk/cleanup.awk b/docs/preprocessor/awk/cleanup.awk deleted file mode 100644 index b49a2cc772c..00000000000 --- a/docs/preprocessor/awk/cleanup.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+EVALUATED'$/ { evaluated = 1 } -/^==>\-EVALUATED$/ { evaluated = 0 } - -!/^((gremlin> ')|==>)[\+\-]EVALUATED(')?$/ { - if ($0 !~ /^gremlin> pb\([0-9]*\); '----'$/) { - if (!evaluated || $0 !~ /^gremlin> [']?:/) { - if (evaluated && $0 ~ /^==>:/) gsub(/^==>/, "gremlin> ") - if (!evaluated || $0 == "==>----") gsub(/^==>/, "") - if (evaluated) { - if ($0 !~ /^WARN /) print - } else if ($0 !~ /^gremlin> pb\([0-9]*\); / && $0 !~ /^gremlin> $/) { - print - } - } - } -} diff --git a/docs/preprocessor/awk/ignore.awk b/docs/preprocessor/awk/ignore.awk deleted file mode 100644 index 99f55e06d90..00000000000 --- a/docs/preprocessor/awk/ignore.awk +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+IGNORE'$/ { ignore = 1 } -{ if (!ignore) print } -/^==>\-IGNORE$/ { ignore = 0 } diff --git a/docs/preprocessor/awk/init-code-blocks.awk b/docs/preprocessor/awk/init-code-blocks.awk deleted file mode 100644 index 8349d6e6410..00000000000 --- a/docs/preprocessor/awk/init-code-blocks.awk +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function capitalize(string) { - return toupper(substr(string, 1, 1)) substr(string, 2) -} - -BEGIN { - delimiter = 0 -} - -/^pb\([0-9]*\); '\[gremlin-/ { - delimiter = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, c, ",") - split(a[1], d, ";") - lang = c[1] - graph = c[2] - print d[1] "; '[source," lang "]'" - print "'+EVALUATED'" - print "'+IGNORE'" - if (graph != "existing") { - if (graph) { - print "graph = TinkerFactory.create" capitalize(graph) "()" - } else { - print "graph = TinkerGraph.open()" - } - print "g = graph.traversal()" - print "marko = g.V().has('name', 'marko').tryNext().orElse(null)" - print "f = new File('/tmp/neo4j')" - print "if (f.exists()) f.deleteDir()" - print "f = new File('/tmp/tinkergraph.kryo')" - print "if (f.exists()) f.deleteDir()" - print ":set max-iteration 100" - } - print "'-IGNORE'" -} - -!/^pb\([0-9]*\); '\[gremlin-/ { - if (delimiter == 2 && !($0 ~ /^pb\([0-9]*\); '----'/)) { - switch (lang) { - default: - print - break - } - } else print -} - -/^pb\([0-9]*\); '----'/ { - if (delimiter == 1) delimiter = 2 - else if (delimiter == 2) { - print "'-EVALUATED'" - delimiter = 0 - } -} diff --git a/docs/preprocessor/awk/language-variants.awk b/docs/preprocessor/awk/language-variants.awk deleted file mode 100644 index 9d9fbf03dff..00000000000 --- a/docs/preprocessor/awk/language-variants.awk +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - lang = null - inCodeBlock = 0 -} - -/^\[source,/ { - delimiter = 1 - split($0, a, ",") - lang = gensub(/]$/, "", 1, a[2]) -} - -/^----$/ { - if (inCodeBlock == 0) inCodeBlock = 1 - else inCodeBlock = 0 -} - -{ if (inCodeBlock) { - switch (lang) { - default: - print - break - } - } else print -} diff --git a/docs/preprocessor/awk/prepare.awk b/docs/preprocessor/awk/prepare.awk deleted file mode 100644 index 860dd4ee2bb..00000000000 --- a/docs/preprocessor/awk/prepare.awk +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - p = 0 - c = "//" -} - -function escape_string(string) { - str = gensub(/\\/, "\\\\\\\\", "g", string) - return gensub(/'/, "\\\\'", "g", str) -} - -function print_string(string) { - print "pb(" p++ "); '" escape_string(string) "'" -} - -function transform_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, " " c c " \\1", "g", code) -} - -function remove_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, "", "g", code) -} - -/^----$/ { - if (inCodeSection) { - if (prepared) { - inCodeSection = 0 - prepared = 0 - } else { - prepared = 1 - } - } - print_string($0) -} - -!/^----$/ { - if (inCodeSection) { - if ($0 ~ /^:/) { - print "'" escape_string(transform_callouts($0)) "'" - print remove_callouts($0) - } else { - print transform_callouts($0) - } - } else { - print_string($0) - } -} - -/^\[gremlin-/ { - inCodeSection = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, l, ",") - lang = l[1] - switch (lang) { - default: - c = "//" - break - } -} - -END { - print_string("// LAST LINE") -} diff --git a/docs/preprocessor/awk/prettify.awk b/docs/preprocessor/awk/prettify.awk deleted file mode 100644 index 522ac631924..00000000000 --- a/docs/preprocessor/awk/prettify.awk +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^==>\/\/\/\/$/ { doPrint = 1 } - -{ - if (inCodeSection && $0 ~ /^\.*[0-9]+> /) { - gsub(/^.{8}/, " ") - } - if (doPrint) print -} - -/^==>\+EVALUATED/ { inCodeSection = 1 } -/^==>\-EVALUATED/ { inCodeSection = 0 } diff --git a/docs/preprocessor/awk/progressbar.awk b/docs/preprocessor/awk/progressbar.awk deleted file mode 100644 index 25109c42e00..00000000000 --- a/docs/preprocessor/awk/progressbar.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - max = 0 - content = "" -} - -/^pb\([0-9]*\); / { - max = gensub(/^pb\(([0-9]*)\); .*/, "\\1", "g", $0) -} -{ content = content "\n" $0 } - -END { - while ((getline line < tpl) > 0) { - print gensub(/TOTAL_LINES/, max, "g", line) - } - print content -} diff --git a/docs/preprocessor/awk/tabify.awk b/docs/preprocessor/awk/tabify.awk deleted file mode 100644 index 7e68c13e5dd..00000000000 --- a/docs/preprocessor/awk/tabify.awk +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function print_tabs(next_id, tabs, blocks) { - - num_tabs = length(tabs) - x = next_id - - print "++++" - print "
" - - for (i = 1; i <= num_tabs; i++) { - title = tabs[i] - print " " - print " " - x++ - } - - for (i = 1; i<= num_tabs; i++) { - print "
" - print "
" - print "++++\n" - print blocks[i] - print "++++" - print "
" - print "
" - } - - print "
" - print "++++\n" -} - -function transform_callouts(code, c) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*\n/, " " c c " \\1\\2\n", "g", code) -} - -BEGIN { - id_part=systime() - status = 0 - next_id = 1 - block[0] = 0 # initialize "blocks" as an array - delete blocks[0] -} - -/^\[gremlin-/ { - status = 1 - lang = gensub(/^\[gremlin-([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 1 -} - -/^\[source,(csharp|groovy|java|javascript|python|go),tab\]/ { - status = 1 - lang = gensub(/^\[source,([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 0 -} - -/^\[source,(csharp|groovy|java|javascript|python|go)\]/ { - if (status == 3) { - status = 1 - lang = gensub(/^\[source,([^\]]+).*/, "\\1", "g", $0) - code = "" - } -} - -! /^\[source,(csharp|groovy|java|javascript|python|go)/ { - if (status == 3 && $0 != "") { - print_tabs(next_id, tabs, blocks) - next_id = next_id + length(tabs) - for (i in tabs) { - delete tabs[i] - delete blocks[i] - } - status = 0 - } -} - -/^----$/ { - if (status == 1) { - status = 2 - } else if (status == 2) { - status = 3 - } -} - -{ if (status == 3) { - if ($0 == "----") { - i = length(blocks) + 1 - if (i == 1 && evaluate == 1) { - tabs[i] = "console (" lang ")" - blocks[i] = code_header code "\n" $0 "\n" - i++ - } - tabs[i] = lang - switch (lang) { - default: - c = "//" - break - } - blocks[i] = "[source," lang "]" transform_callouts(code, c) "\n" $0 "\n" - } - } else { - if (status == 0) print - else if (status == 1) code_header = gensub(/,tab/, "", "g", $0) - else code = code "\n" $0 - } -} - -END { - # EOF - if (status == 3) { - print_tabs(next_id, tabs, blocks) - } -} diff --git a/docs/preprocessor/install-plugins.groovy b/docs/preprocessor/install-plugins.groovy deleted file mode 100644 index 0edbfdbd48e..00000000000 --- a/docs/preprocessor/install-plugins.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * @author Daniel Kuppitz (http://gremlin.guru) - */ -import org.apache.tinkerpop.gremlin.groovy.util.Artifact -import org.apache.tinkerpop.gremlin.groovy.util.DependencyGrabber - -installPlugin = { def artifact -> - def classLoader = new groovy.lang.GroovyClassLoader() - def extensionPath = new File(System.getProperty("user.dir"), "ext") - try { - System.err.print(" * ${artifact.getArtifact()} ... ") - new DependencyGrabber(classLoader, extensionPath).copyDependenciesToPath(artifact) - System.err.println("done") - } catch (Exception e) { - System.err.println("failed") - System.err.println() - System.err.println(e.getMessage()) - e.printStackTrace() - System.exit(1) - } -} - -:plugin use tinkerpop.sugar -:plugin use tinkerpop.credentials -System.err.println("done") diff --git a/docs/preprocessor/install-plugins.sh b/docs/preprocessor/install-plugins.sh deleted file mode 100755 index ba882132076..00000000000 --- a/docs/preprocessor/install-plugins.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TP_VERSION=$2 -TMP_DIR=$3 -INSTALL_TEMPLATE="docs/preprocessor/install-plugins.groovy" -INSTALL_FILE="${TMP_DIR}/install-plugins.groovy" - -plugins=("hadoop-gremlin" "spark-gremlin" "neo4j-gremlin" "sparql-gremlin") -# plugins=() -pluginsCount=${#plugins[@]} - -i=0 - -cp ${INSTALL_TEMPLATE} ${INSTALL_FILE} - -while [ ${i} -lt ${pluginsCount} ]; do - pluginName=${plugins[$i]} - className="" - for part in $(tr '-' '\n' <<< ${pluginName}); do - className="${className}$(tr '[:lower:]' '[:upper:]' <<< ${part:0:1})${part:1}" - done - pluginClassFile=$(find . -name "${className}Plugin.java") - pluginClass=`sed -e 's@.*src/main/java/@@' -e 's/\.java$//' <<< ${pluginClassFile} | tr '/' '.'` - installed=`grep -c "${pluginClass}" ${CONSOLE_HOME}/ext/plugins.txt` - if [ ${installed} -eq 0 ]; then - echo "installPlugin(new Artifact(\"org.apache.tinkerpop\", \"${pluginName}\", \"${TP_VERSION}\"))" >> ${INSTALL_FILE} - echo "${pluginName}" >> ${TMP_DIR}/plugins.dir - echo "${pluginClass}" >> ${TMP_DIR}/plugins.txt - else - echo " * skipping ${pluginName} (already installed)" - fi - ((i++)) -done - -echo "System.exit(0)" >> ${INSTALL_FILE} -echo -ne " * tinkerpop-sugar ... " - -pushd ${CONSOLE_HOME} > /dev/null - -mkdir -p ~/.java/.userPrefs -chmod 700 ~/.java/.userPrefs - -bin/gremlin.sh -e ${INSTALL_FILE} > /dev/null - -if [ ${PIPESTATUS[0]} -ne 0 ]; then - popd > /dev/null - exit 1 -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - cat ${TMP_DIR}/plugins.txt >> ${CONSOLE_HOME}/ext/plugins.txt -fi - -popd > /dev/null diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh deleted file mode 100755 index 9ae43c2f8d0..00000000000 --- a/docs/preprocessor/preprocess-file.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -TP_HOME=`pwd` -CONSOLE_HOME=$1 -AWK_SCRIPTS="${TP_HOME}/docs/preprocessor/awk" - -IFS=',' read -r -a DRYRUN_DOCS <<< "$2" -IFS=',' read -r -a FULLRUN_DOCS <<< "$3" - -dryRun () { - local doc - yes=0 - no=1 - doDryRun=${no} - if [ "${DRYRUN_DOCS}" == "*" ]; then - doDryRun=${yes} - else - for doc in "${DRYRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${yes} - break - fi - done - fi - if [ ${doDryRun} ]; then - for doc in "${FULLRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${no} - break - fi - done - fi - return ${doDryRun} -} - -input=$4 -output=`sed 's@/docs/src/@/target/postprocess-asciidoc/@' <<< "${input}"` - -SKIP= -if dryRun ${input}; then - SKIP=1 -fi - -mkdir -p `dirname ${output}` - -if hash stdbuf 2> /dev/null; then - lb="stdbuf -oL" -else - lb="" -fi - -trap cleanup INT - -function cleanup { - if [ -f "${output}" ]; then - if [ `wc -l "${output}" | awk '{print $1}'` -gt 0 ]; then - echo -e "\n\e[1mLast 10 lines of ${output}:\e[0m\n" - tail -n10 ${output} - echo - echo "Opening ${output} for full inspection" - sleep 5 - less ${output} - fi - fi - rm -rf ${output} ${CONSOLE_HOME}/.ext - exit 255 -} - -function processed { - echo -ne "\r progress: [====================================================================================================] 100%\n" -} - -echo -echo " * source: ${input}" -echo " target: ${output}" -echo -ne " progress: initializing" - -if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then - if [ ${output} -nt ${input} ]; then - processed - exit 0 - fi - pushd "${CONSOLE_HOME}" > /dev/null - - doc=`basename ${input} .asciidoc` - - case "${doc}" in - "implementations-neo4j") - # deactivate Spark plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/spark-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'SparkGremlinPlugin' > .ext/plugins.txt - ;; - "implementations-hadoop-start" | "implementations-hadoop-end" | "implementations-spark" | "olap-spark-yarn") - # deactivate Neo4j plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/neo4j-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'Neo4jGremlinPlugin' > .ext/plugins.txt - ;; - "gremlin-variants") - # deactivate plugin to prevent version conflicts - mkdir .ext - mv ext/neo4j-gremlin .ext/ - mv ext/spark-gremlin .ext/ - mv ext/hadoop-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -v 'Neo4jGremlinPlugin\|SparkGremlinPlugin\|HadoopGremlinPlugin' > .ext/plugins.txt - ;; - esac - - if [ -d ".ext" ]; then - mv .ext/plugins.txt ext/ - fi - - sed 's/\t/ /g' ${input} | - awk -f ${AWK_SCRIPTS}/tabify.awk | - awk -f ${AWK_SCRIPTS}/prepare.awk | - awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" | - awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | - HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh | ${TP_HOME}/docs/preprocessor/control-characters.sh | - ${lb} awk -f ${AWK_SCRIPTS}/ignore.awk | - ${lb} awk -f ${AWK_SCRIPTS}/prettify.awk | - ${lb} awk -f ${AWK_SCRIPTS}/cleanup.awk | - ${lb} awk -f ${AWK_SCRIPTS}/language-variants.awk > ${output} - - # check exit code for each of the previously piped commands - ps=(${PIPESTATUS[@]}) - for i in {0..9}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - - if [ -d ".ext" ]; then - mv .ext/plugins.all ext/plugins.txt - mv .ext/* ext/ - rm -r .ext/ - fi - - if [ ${ec} -eq 0 ]; then - tail -n1 ${output} | grep -F '// LAST LINE' > /dev/null - ec=$? - fi - - if [ ${ec} -eq 0 ]; then - processed - fi - - echo - popd > /dev/null - if [ ${ec} -ne 0 ]; then - cleanup - fi -else - cp ${input} ${output} - processed -fi diff --git a/docs/preprocessor/preprocess.sh b/docs/preprocessor/preprocess.sh deleted file mode 100755 index 3c05d946e63..00000000000 --- a/docs/preprocessor/preprocess.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -DRYRUN_DOCS="$1" -FULLRUN_DOCS="$2" - -pushd "$(dirname $0)/../.." > /dev/null - -if [ "${DRYRUN_DOCS}" != "*" ]; then - - if [ ! -f bin/gremlin.sh ]; then - echo "Gremlin REPL is not available. Cannot preprocess AsciiDoc files." - popd > /dev/null - exit 1 - fi - - for daemon in "NameNode" "DataNode" "ResourceManager" "NodeManager" - do - running=`jps | cut -d ' ' -f2 | grep -c ${daemon}` - if [ ${running} -eq 0 ]; then - echo "Hadoop is not running, be sure to start it before processing the docs." - exit 1 - fi - done - - netstat -an | awk '{print $4}' | grep -o '[0-9]*$' | grep '\b8182\b' > /dev/null && { - echo "The port 8182 is required for Gremlin Server, but it is already in use. Be sure to close the application that currently uses the port before processing the docs." - exit 1 - } - - if [ -e /tmp/neo4j ]; then - echo "The directory '/tmp/neo4j' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi - - if [ -e /tmp/tinkergraph.kryo ]; then - echo "The file '/tmp/tinkergraph.kryo' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi -fi - -function directory { - d1=`pwd` - cd $1 - d2=`pwd` - cd $d1 - echo "$d2" -} - -mkdir -p target/postprocess-asciidoc/tmp -mkdir -p target/postprocess-asciidoc/logs -cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - -TP_HOME=`pwd` -CONSOLE_HOME=`directory "${TP_HOME}/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone"` -PLUGIN_DIR="${CONSOLE_HOME}/ext" -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) -TMP_DIR="/tmp/tp-docs-preprocessor" - -mkdir -p "${TMP_DIR}" - -HISTORY_FILE=".gremlin_groovy_history" -[ -f ~/${HISTORY_FILE} ] && cp ~/${HISTORY_FILE} ${TMP_DIR} - -pushd gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone > /dev/null -bin/gremlin-server.sh conf/gremlin-server-modern.yaml > ${TP_HOME}/target/postprocess-asciidoc/logs/gremlin-server.log 2>&1 & -GREMLIN_SERVER_PID=$! -popd > /dev/null - -function cleanup() { - echo -ne "\r\n\n" - docs/preprocessor/uninstall-plugins.sh "${CONSOLE_HOME}" "${TMP_DIR}" - [ -f ${TMP_DIR}/plugins.txt.orig ] && mv ${TMP_DIR}/plugins.txt.orig ${CONSOLE_HOME}/ext/plugins.txt - find ${TP_HOME}/docs/src/ -name "*.asciidoc.groovy" | xargs rm -f - [ -f ${TMP_DIR}/${HISTORY_FILE} ] && mv ${TMP_DIR}/${HISTORY_FILE} ~/ - rm -rf ${TMP_DIR} - kill ${GREMLIN_SERVER_PID} &> /dev/null - popd &> /dev/null -} - -trap cleanup EXIT - -if [ "${DRYRUN_DOCS}" != "*" ] || [ ! -z "${FULLRUN_DOCS}" ]; then - - # install plugins - echo - echo "==========================" - echo "+ Installing Plugins +" - echo "==========================" - echo - cp ${CONSOLE_HOME}/ext/plugins.txt ${TMP_DIR}/plugins.txt.orig - docs/preprocessor/install-plugins.sh "${CONSOLE_HOME}" "${TP_VERSION}" "${TMP_DIR}" - - if [ $? -ne 0 ]; then - exit 1 - else - echo - fi - -fi - -# process *.asciidoc files -COLS=${COLUMNS} -[[ ${COLUMNS} -lt 240 ]] && stty cols 240 - -tput rmam - -echo -echo "============================" -echo "+ Processing AsciiDocs +" -echo "============================" - -ec=0 -for subdir in $(find "${TP_HOME}/docs/src/" -name index.asciidoc | xargs -n1 dirname) -do - find "${subdir}" -maxdepth 1 -name "*.asciidoc" | - xargs -n1 basename | - xargs -n1 -I {} echo "echo -ne {}' '; (grep -n {} ${subdir}/index.asciidoc || echo 0) | head -n1 | cut -d ':' -f1" | /bin/bash | sort -nk2 | cut -d ' ' -f1 | - xargs -n1 -I {} echo "${subdir}/{}" | - xargs -n1 ${TP_HOME}/docs/preprocessor/preprocess-file.sh "${CONSOLE_HOME}" "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - - ps=(${PIPESTATUS[@]}) - for i in {0..7}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - [ ${ec} -eq 0 ] || break -done - -tput smam -[[ "${COLUMNS}" != "" ]] && stty cols ${COLS} - -rm -rf /tmp/neo4j /tmp/tinkergraph.kryo - -[ ${ec} -eq 0 ] || exit 1 - -echo diff --git a/docs/preprocessor/uninstall-plugins.sh b/docs/preprocessor/uninstall-plugins.sh deleted file mode 100755 index 6353fe50f71..00000000000 --- a/docs/preprocessor/uninstall-plugins.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TMP_DIR=$2 - -if [ -f "${TMP_DIR}/plugins.dir" ]; then - for pluginDirectory in $(cat ${TMP_DIR}/plugins.dir); do - rm -rf ${CONSOLE_HOME}/ext/${pluginDirectory} - done -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - for className in $(cat ${TMP_DIR}/plugins.txt); do - sed -e "/${className}/d" ${CONSOLE_HOME}/ext/plugins.txt > ${CONSOLE_HOME}/ext/plugins.txt. - mv ${CONSOLE_HOME}/ext/plugins.txt. ${CONSOLE_HOME}/ext/plugins.txt - done -fi diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 5756bc8da88..9ff47464cd1 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -122,95 +122,26 @@ an issue when working with SNAPSHOT dependencies. [[documentation-environment]] === Documentation Environment -The documentation generation process is not Maven-based and uses shell scripts to process the project's asciidoc. The -scripts should work on Mac and Linux. Javadocs should be built using Java 11. +The core of the documentation generation process is an link:https://asciidoctor.org/[AsciidoctorJ] extension +(`tools/tinkerpop-docs`) which walks each AsciiDoc book, executes the `[gremlin-groovy]` code blocks against a long-lived +Gremlin Console subprocess, and renders the console output as tabbed HTML. The orchestration script +`bin/process-docs.sh` wraps this and acts as the entrypoint: it validates the Gremlin Console and Gremlin Server +distributions, installs the required plugins into the console, starts a Gremlin Server (for the `:remote` examples) and +a Gephi mock, then invokes Maven to run the extension. Javadocs should be built using Java 11. -TIP: We recommend performing documentation generation on Linux. For the scripts to work on Mac, you will need to -install GNU versions of the utility programs via `homebrew`, e.g.`grep`, `awk`, `sed`, `findutils`, and `diffutils`. +Before generating documentation, build the build all of the dependencies that the process consumes: -To generate documentation, it is required that link:https://hadoop.apache.org[Hadoop 3.3.x] is running in -link:https://hadoop.apache.org/docs/r3.3.1/hadoop-project-dist/hadoop-common/SingleCluster.html#Pseudo-Distributed_Operation[pseudo-distributed] -mode. Be sure to set the `HADOOP_GREMLIN_LIBS` environment variable as described in the -link:https://tinkerpop.apache.org/docs/x.y.z/reference/#hadoop-gremlin[reference documentation]. It is also important -to set the `CLASSPATH` to point at the directory containing the Hadoop configuration files, like `mapred-site.xml`. - -The `/etc/hadoop/yarn-site.xml` file prefers this configuration over the one provided in the Hadoop documentation -referenced above: - -[source,xml] ----- - - - yarn.nodemanager.aux-services - mapreduce_shuffle - - - yarn.nodemanager.vmem-check-enabled - false - - - yarn.nodemanager.vmem-pmem-ratio - 4 - - ----- - -The `/etc/hadoop/mapred-site.xml` file prefers the following configuration: - -[source,xml] ----- - - - mapreduce.framework.name - yarn - - - mapred.map.tasks - 4 - - - mapred.reduce.tasks - 4 - - - mapreduce.job.counters.limit - 1000 - - - mapreduce.jobtracker.address - localhost:9001 - - - mapreduce.map.memory.mb - 2048 - - - mapreduce.reduce.memory.mb - 4096 - - - mapreduce.map.java.opts - -Xmx2048m - - - mapreduce.reduce.java.opts - -Xmx4096m - - ----- - -Also note that link:http://www.grymoire.com/Unix/Awk.html[awk] version `4.0.1` is required for documentation generation. -The link:https://tinkerpop.apache.org/docs/x.y.z/recipes/#olap-spark-yarn[YARN recipe] also uses the `zip` program to -create an archive so that needs to be installed, too, if you don't have it already. - -The Hadoop 3.3.x installation instructions call for installing `pdsh` but installing that seems to cause permission -problems when executing `sbin/start-dfs.sh`. Skipping that prerequisite seems to solve the problem. +[source,text] +mvn clean install -pl :gremlin-console,:gremlin-server:spark-gremlin,:neo4j-gremlin -am -DskipTests -DincludeNeo4j -Documentation can be generated locally with: +Documentation can then be generated locally with: [source,text] bin/process-docs.sh +A `--dryRun` option renders the books without starting a console or server and without executing any code blocks, +which is useful for quickly checking AsciiDoc/formatting changes. + Documentation is generated to the `target/docs` directory. It is also possible to generate documentation locally with Docker. `docker/build.sh -d`. @@ -219,14 +150,36 @@ failed`. It often helps in this case to delete the directories for the dependenc in the `.m2` (`~/.m2/`) and in the `grapes` (`~/.groovy/grapes/`) cache. E.g., if the error is about `asm#asm;3.2!asm.jar`, then remove the `asm/asm` sub directory in both directories. -NOTE: Unexpected failures with OLAP often point to a jar conflict that arises in scenarios where Hadoop or Spark -dependencies (or other dependencies for that matter) are modified and conflict. It is not picked up by the enforcer -plugin because the inconsistency arises through plugin installation in Gremlin Console at document generation time. -Making adjustments to the various paths by way of the `` on the jar given the functionality provided -by the `DependencyGrabber` class which allows you to manipulate (typically deleting conflicting files from `/lib` and -`/plugin`) plugin loading will usually resolve it, though it could also be a more general environmental problem with -Spark or Hadoop. The easiest way to see the error is to simply run the examples in the Gremlin Console which more -plainly displays the error than the failure of the documentation generation process. +NOTE: Unexpected failures with OLAP often point to a jar conflict that arises when Hadoop, Spark, or Neo4j +dependencies are modified and collide on the console's classpath. It is not picked up by the enforcer plugin because +the inconsistency arises through plugin installation in the Gremlin Console at document generation time. The most +common case -- Neo4j (Scala 2.11) and Spark (Scala 2.12) -- is handled by the per-book plugin exclusion mechanism +described in <>. For other conflicts, the `` (`Gremlin-Plugin-Paths` / +`Gremlin-Lib-Paths`) on the plugin jar control how the `DependencyGrabber` lays jars out under `ext//plugin` +and `ext//lib`, which can be adjusted to resolve ordering or duplicate-jar problems. The easiest way to see +the underlying error is to run the offending example directly in the Gremlin Console, which displays it more plainly +than the documentation build does. + +[[docs-plugin-exclusions]] +==== Per-book Plugin Exclusions + +Some plugins cannot share the documentation console's classpath. Most notably, Neo4j 3.4 requires Scala 2.11 while +Spark requires Scala 2.12, so activating both at once causes runtime failures (e.g. `NoSuchMethodError: +scala.Product.$init$`). To avoid this, a book can restart the console with certain plugins removed by declaring the +`gremlin-docs-plugins-exclude` attribute (a comma-separated list of plugin directory names) on a section heading: + +[source,asciidoc] +---- +[gremlin-docs-plugins-exclude="neo4j-gremlin"] +==== SparkGraphComputer +---- + +When the `GremlinTreeprocessor` reaches a section carrying this attribute, it closes the current console, moves the +named `ext/` directories aside (and updates `ext/plugins.txt`), then starts a fresh console so the excluded +plugins are off the classpath. The exclusion is latched: it remains in effect for subsequent sections until another +section declares a different set, so each conflicting chapter declares its complete exclusion set. This replaces the +old preprocessor's per-file plugin juggling; `bin/process-docs.sh` installs each plugin's dependencies into +`ext//plugin/` (not the shared `lib/`) precisely so they can be toggled this way. To generate the web site locally, there is no need for any of the above infrastructure. Site generation is a simple shell script: @@ -525,7 +478,7 @@ mvn -Dmaven.javadoc.skip=true --projects tinkergraph-gremlin test ** Build AsciiDocs (but don't evaluate code blocks): `bin/process-docs.sh --dryRun` ** Build AsciiDocs (but don't evaluate code blocks in specific files): `bin/process-docs.sh --dryRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` ** Build AsciiDocs (but evaluate code blocks only in specific files): `bin/process-docs.sh --fullRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` -** Process a single AsciiDoc file: +pass:[docs/preprocessor/preprocess-file.sh `pwd`/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone "" "*" `pwd`/docs/src/xyz.asciidoc]+ +** Process docs in dry-run mode (no console required): `bin/process-docs.sh --dryRun` * Build JavaDocs/JSDoc: `mvn process-resources -Djavadoc` ** Javadoc to `target/site/apidocs` directory ** JSDoc to the `gremlin-javascript/src/main/javascript/gremlin-javascript/doc/` directory diff --git a/docs/src/dev/developer/for-committers.asciidoc b/docs/src/dev/developer/for-committers.asciidoc index 5d40a18757d..3e03b9a1b14 100644 --- a/docs/src/dev/developer/for-committers.asciidoc +++ b/docs/src/dev/developer/for-committers.asciidoc @@ -873,9 +873,9 @@ of the Apache "Licensing How-to" for more information. The documentation for TinkerPop is stored in the git repository in `docs/src/` and are then split into several subdirectories, each representing a "book" (or its own publishable body of work). If a new AsciiDoc file is added to -a book, then it should also be included in the `index.asciidoc` file for that book, otherwise the preprocessor will -ignore it. Likewise, if a whole new book (subdirectory) is added, it must include an `index.asciidoc` file to be -recognized by the AsciiDoc preprocessor. +a book, then it should also be included in the `index.asciidoc` file for that book, otherwise the documentation build +will ignore it. Likewise, if a whole new book (subdirectory) is added, it must include an `index.asciidoc` file to be +recognized by the documentation build. Adding a book also requires a change to the root `pom.xml` file. Find the "asciidoc" Maven profile and add a new `` to the `asciidoctor-maven-plugin` configuration. For each book in `docs/src/`, there should be a diff --git a/docs/src/recipes/olap-spark-yarn.asciidoc b/docs/src/recipes/olap-spark-yarn.asciidoc index f8a04c121db..12ea3c77d02 100644 --- a/docs/src/recipes/olap-spark-yarn.asciidoc +++ b/docs/src/recipes/olap-spark-yarn.asciidoc @@ -89,26 +89,44 @@ $ hdfs dfs -put data/tinkerpop-modern.kryo . $ . bin/spark-yarn.sh ---- -[gremlin-groovy] +[source,groovy] ---- -hadoop = System.getenv('HADOOP_HOME') -hadoopConfDir = System.getenv('HADOOP_CONF_DIR') -archive = 'spark-gremlin.zip' -archivePath = "/tmp/$archive" -['bash', '-c', "rm -f $archivePath; cd ext/spark-gremlin/lib && zip $archivePath *.jar"].execute().waitFor() -conf = new Configurations().properties(new File('conf/hadoop/hadoop-gryo.properties')) -conf.setProperty('spark.master', 'yarn') -conf.setProperty('spark.submit.deployMode', 'client') -conf.setProperty('spark.yarn.archive', "$archivePath") -conf.setProperty('spark.yarn.appMasterEnv.CLASSPATH', "./__spark_libs__/*:$hadoopConfDir") -conf.setProperty('spark.executor.extraClassPath', "./__spark_libs__/*:$hadoopConfDir") -conf.setProperty('spark.driver.extraLibraryPath', "$hadoop/lib/native:$hadoop/lib/native/Linux-amd64-64") -conf.setProperty('spark.executor.extraLibraryPath', "$hadoop/lib/native:$hadoop/lib/native/Linux-amd64-64") -conf.setProperty('gremlin.spark.persistContext', 'true') -hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo') -graph = GraphFactory.open(conf) -g = traversal().withEmbedded(graph).withComputer(SparkGraphComputer) -g.V().group().by(values('name')).by(both().count()) +gremlin> hadoop = System.getenv('HADOOP_HOME') +==>/usr/local/lib/hadoop-3.3.1 +gremlin> hadoopConfDir = System.getenv('HADOOP_CONF_DIR') +==>/usr/local/lib/hadoop-3.3.1/etc/hadoop +gremlin> archive = 'spark-gremlin.zip' +==>spark-gremlin.zip +gremlin> archivePath = "/tmp/$archive" +==>/tmp/spark-gremlin.zip +gremlin> ['bash', '-c', "rm -f $archivePath; cd ext/spark-gremlin/lib && zip $archivePath *.jar"].execute().waitFor() +==>0 +gremlin> conf = new Configurations().properties(new File('conf/hadoop/hadoop-gryo.properties')) +==>org.apache.commons.configuration2.PropertiesConfiguration@5b3bb1f7 +gremlin> conf.setProperty('spark.master', 'yarn') +==>null +gremlin> conf.setProperty('spark.submit.deployMode', 'client') +==>null +gremlin> conf.setProperty('spark.yarn.archive', "$archivePath") +==>null +gremlin> conf.setProperty('spark.yarn.appMasterEnv.CLASSPATH', "./__spark_libs__/*:$hadoopConfDir") +==>null +gremlin> conf.setProperty('spark.executor.extraClassPath', "./__spark_libs__/*:$hadoopConfDir") +==>null +gremlin> conf.setProperty('spark.driver.extraLibraryPath', "$hadoop/lib/native:$hadoop/lib/native/Linux-amd64-64") +==>null +gremlin> conf.setProperty('spark.executor.extraLibraryPath', "$hadoop/lib/native:$hadoop/lib/native/Linux-amd64-64") +==>null +gremlin> conf.setProperty('gremlin.spark.persistContext', 'true') +==>null +gremlin> hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo') +==>null +gremlin> graph = GraphFactory.open(conf) +==>hadoopgraph[gryoinputformat->gryooutputformat] +gremlin> g = traversal().withEmbedded(graph).withComputer(SparkGraphComputer) +==>graphtraversalsource[hadoopgraph[gryoinputformat->gryooutputformat], sparkgraphcomputer] +gremlin> g.V().group().by(values('name')).by(both().count()) +==>[ripple:1,peter:1,vadas:1,josh:3,lop:3,marko:3] ---- If you run into exceptions, you will have to dig into the logs. You can do this from the command line with diff --git a/docs/src/recipes/traversal-induced-values.asciidoc b/docs/src/recipes/traversal-induced-values.asciidoc index 2d058a5ee22..162651996fd 100644 --- a/docs/src/recipes/traversal-induced-values.asciidoc +++ b/docs/src/recipes/traversal-induced-values.asciidoc @@ -36,7 +36,7 @@ obvious to any programmer - use a variable: [gremlin-groovy,modern] ---- vMarko = g.V().has('name','marko').next() -g.V(vMarko).out('knows').has('age', gt(marko.value('age'))).values('name') +g.V(vMarko).out('knows').has('age', gt(vMarko.value('age'))).values('name') ---- The downside to this approach is that it takes two separate traversals to answer the question. Ideally, there should diff --git a/docs/src/reference/gremlin-variants.asciidoc b/docs/src/reference/gremlin-variants.asciidoc index 4d9c3f606c3..e3336139aa2 100644 --- a/docs/src/reference/gremlin-variants.asciidoc +++ b/docs/src/reference/gremlin-variants.asciidoc @@ -18,6 +18,10 @@ under the License. //// anchor:gremlin-variants[] +// This book exercises only remote driver connections, but its predecessors leave Neo4j/Spark/Hadoop +// plugins active. Exclude all three to restart the console with a clean classpath and avoid the +// Scala 2.11/2.12 (and related) conflicts those plugins introduce. +[gremlin-docs-plugins-exclude="neo4j-gremlin,spark-gremlin,hadoop-gremlin"] [[gremlin-drivers-variants]] = Gremlin Drivers and Variants diff --git a/docs/src/reference/implementations-hadoop-end.asciidoc b/docs/src/reference/implementations-hadoop-end.asciidoc index 2649c006371..338792d6fee 100644 --- a/docs/src/reference/implementations-hadoop-end.asciidoc +++ b/docs/src/reference/implementations-hadoop-end.asciidoc @@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +// Continues the Spark/Hadoop OLAP examples (Scala 2.12), so exclude neo4j-gremlin (Scala 2.11) +// to keep it off the console's flat classpath. +[gremlin-docs-plugins-exclude="neo4j-gremlin"] === Input/Output Formats image:adjacency-list.png[width=300,float=right] Hadoop-Gremlin provides various I/O formats -- i.e. Hadoop @@ -160,12 +163,12 @@ The results of any OLAP operation are stored in HDFS accessible via `hdfs`. For ---- graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties') graph.compute(SparkGraphComputer).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey('clusterCount').create()).submit().get(); -hdfs.ls() +hdfs.ls('tinkerpop-modern.kryo') hdfs.ls('output') hdfs.head('output', GryoInputFormat) hdfs.head('output', 'clusterCount', SequenceFileInputFormat) hdfs.rm('output') -hdfs.ls() +hdfs.ls('tinkerpop-modern.kryo') ---- [[interacting-with-spark]] diff --git a/docs/src/reference/implementations-hadoop-start.asciidoc b/docs/src/reference/implementations-hadoop-start.asciidoc index 31d08daab7e..686eaaf47ee 100644 --- a/docs/src/reference/implementations-hadoop-start.asciidoc +++ b/docs/src/reference/implementations-hadoop-start.asciidoc @@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +// Spark requires Scala 2.12 while Neo4j 3.4 requires Scala 2.11; they cannot share the console's +// flat classpath, so exclude neo4j-gremlin to restart the console without Neo4j's jars. +[gremlin-docs-plugins-exclude="neo4j-gremlin"] [[hadoop-gremlin]] == Hadoop-Gremlin @@ -148,7 +151,7 @@ which is the OLAP Gremlin machine. [gremlin-groovy] ---- hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo') -hdfs.ls() +hdfs.ls('tinkerpop-modern.kryo') graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties') g = traversal().withEmbedded(graph) g.V().count() diff --git a/docs/src/reference/implementations-neo4j.asciidoc b/docs/src/reference/implementations-neo4j.asciidoc index d784b2076b7..e5dfbb0f9b9 100644 --- a/docs/src/reference/implementations-neo4j.asciidoc +++ b/docs/src/reference/implementations-neo4j.asciidoc @@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +// Neo4j 3.4 requires Scala 2.11; Spark requires Scala 2.12. They cannot coexist on the console's +// flat classpath, so exclude spark-gremlin here to restart the console without Spark's jars. +[gremlin-docs-plugins-exclude="spark-gremlin"] [[neo4j-gremlin]] == Neo4j-Gremlin (Deprecated) diff --git a/docs/src/reference/implementations-spark.asciidoc b/docs/src/reference/implementations-spark.asciidoc index 77140af2ab3..e1e1edc53e9 100644 --- a/docs/src/reference/implementations-spark.asciidoc +++ b/docs/src/reference/implementations-spark.asciidoc @@ -16,6 +16,9 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +// SparkGraphComputer requires Scala 2.12 while Neo4j 3.4 requires Scala 2.11; they cannot share +// the console's flat classpath, so exclude neo4j-gremlin to restart the console without Neo4j's jars. +[gremlin-docs-plugins-exclude="neo4j-gremlin"] [[sparkgraphcomputer]] ==== SparkGraphComputer diff --git a/docs/src/reference/index.asciidoc b/docs/src/reference/index.asciidoc index 98f50906a9d..1bd88585746 100644 --- a/docs/src/reference/index.asciidoc +++ b/docs/src/reference/index.asciidoc @@ -43,9 +43,10 @@ include::implementations-intro.asciidoc[] include::implementations-tinkergraph.asciidoc[] include::implementations-neo4j.asciidoc[] -// the hadoop section is split into parts because of serialization issues that are encountered when trying -// to generate graph/spark without restarting the console and currently the only way to force a restart of the -// console is to have a new asciidoc page. +// The hadoop section is split into parts so the Neo4j/Spark plugins (incompatible Scala versions) +// are not active at the same time. Console restarts are now driven by the +// :gremlin-docs-plugins-exclude: attribute on each chapter heading (see implementations-neo4j, +// implementations-hadoop-start, implementations-spark) rather than by page boundaries. include::implementations-hadoop-start.asciidoc[] include::implementations-spark.asciidoc[] include::implementations-hadoop-end.asciidoc[] diff --git a/pom.xml b/pom.xml index 64fb4c4f143..f8319a7cd9d 100644 --- a/pom.xml +++ b/pom.xml @@ -972,6 +972,15 @@ limitations under the License. ${project.basedir}/target/docs/htmlsingle ${asciidoc.input.dir}/stylesheets + + + + + false + + + ${project.basedir}/target/asciidoctor-gems @@ -1038,6 +1047,13 @@ limitations under the License. org.asciidoctor asciidoctor-maven-plugin false + + + org.apache.tinkerpop + tinkerpop-docs + ${project.version} + + home @@ -1072,6 +1088,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1099,6 +1119,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1126,6 +1150,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1153,6 +1181,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1180,6 +1212,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1207,6 +1243,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1234,6 +1274,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1261,6 +1305,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1286,6 +1334,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1312,6 +1364,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1338,6 +1394,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} @@ -1363,6 +1423,10 @@ limitations under the License. ${project.basedir} shared ${project.basedir}/docs/src + ${gremlin.docs.console.home} + ${gremlin.docs.hadoop.libs} + ${gremlin.docs.dryrun} + ${project.version} diff --git a/tools/tinkerpop-docs/pom.xml b/tools/tinkerpop-docs/pom.xml new file mode 100644 index 00000000000..a327ca4089c --- /dev/null +++ b/tools/tinkerpop-docs/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + org.apache.tinkerpop + tinkerpop + 3.7.7-SNAPSHOT + ../../pom.xml + + tinkerpop-docs + Apache TinkerPop :: Docs Extension + AsciidoctorJ extension for processing gremlin code blocks in TinkerPop documentation + jar + + + 2.5.13 + + + + + org.asciidoctor + asciidoctorj + ${asciidoctorj.version} + provided + + + junit + junit + ${junit.version} + test + + + org.hamcrest + hamcrest + ${hamcrest.version} + test + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.0 + + + 11 + + + + + diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java new file mode 100644 index 00000000000..946c97c6ac7 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleRestartHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.io.IOException; +import java.util.List; + +/** + * Callback interface invoked by the {@link GremlinTreeprocessor} when the document signals + * that the console needs to be restarted with certain plugins excluded. + *

+ * The orchestration layer implements this to close the current console, toggle/rename plugin + * directories, and start a new console instance. + */ +@FunctionalInterface +public interface ConsoleRestartHandler { + + /** + * Called when the document attribute {@code :gremlin-docs-plugins-exclude:} is detected + * with a changed value. The handler should close the current console, exclude the specified + * plugin directories, and start a new console. + * + * @param excludedPlugins list of plugin directory names to exclude (e.g., "neo4j-gremlin") + * @throws IOException if the restart operation fails + */ + void onRestart(List excludedPlugins) throws IOException; +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java new file mode 100644 index 00000000000..2ae49ee26ae --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinConsole.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.file.Path; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * Manages a long-running Gremlin Console subprocess, sending statements and capturing output. + *

+ * Uses prompt-based boundary detection: reads stdout until the {@code gremlin>} prompt appears. + * Automatically dismisses {@code Display stack trace?} error prompts on stderr. + */ +public class GremlinConsole implements Closeable { + + static final String PROMPT = "gremlin>"; + static final String ERROR_PROMPT = "Display stack trace?"; + private static final long DEFAULT_TIMEOUT_MS = 90_000; + private static final int EOF = -1; + + private final Process process; + private final Writer stdin; + private final BufferedReader stdout; + private final BufferedReader stderr; + private final Object stderrStdinLock = new Object(); + private final Thread errorDismisser; + private final Thread stdoutReaderThread; + private final BlockingQueue stdoutQueue = new LinkedBlockingQueue<>(); + private final long timeoutMs; + private volatile boolean running; + private boolean inEscape; + private volatile boolean errorPromptSeen; + private volatile String lastErrorText = ""; + private final StringBuilder errorCapture = new StringBuilder(); + private static final int MAX_ERROR_CAPTURE = 8192; + + /** + * Creates a new GremlinConsole by starting the {@code bin/gremlin.sh} subprocess. + * + * @param tinkerpopHome path to the TinkerPop distribution home directory + */ + public GremlinConsole(final Path tinkerpopHome) throws IOException, ConsoleTimeoutException { + this(new ProcessBuilder(tinkerpopHome.resolve("bin/gremlin.sh").toString()) + .redirectErrorStream(false) + .start(), DEFAULT_TIMEOUT_MS); + } + + /** + * Package-private constructor for testing with a pre-built process. + */ + GremlinConsole(final Process process, final long timeoutMs) throws IOException, ConsoleTimeoutException { + this.process = process; + this.stdin = new OutputStreamWriter(process.getOutputStream()); + this.stdout = new BufferedReader(new InputStreamReader(process.getInputStream())); + this.stderr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + this.timeoutMs = timeoutMs; + this.running = true; + + this.stdoutReaderThread = new Thread(this::readStdout, "gremlin-console-stdout-reader"); + this.stdoutReaderThread.setDaemon(true); + this.stdoutReaderThread.start(); + + this.errorDismisser = new Thread(this::dismissErrorPrompts, "gremlin-console-error-dismisser"); + this.errorDismisser.setDaemon(true); + this.errorDismisser.start(); + + // Wait for initial prompt + readUntilPrompt(); + } + + /** + * Sends a statement to the console and returns the output (excluding the prompt line). + * + * @param statement the Gremlin statement to execute + * @return the console output for this statement + * @throws IOException if an I/O error occurs + * @throws ConsoleTimeoutException if the prompt is not received within the timeout period + * @throws GremlinExecutionException if the statement produced a Gremlin error + */ + public String execute(final String statement) throws IOException, ConsoleTimeoutException { + synchronized (stderrStdinLock) { + errorPromptSeen = false; + lastErrorText = ""; + errorCapture.setLength(0); + } + stdin.write(statement + "\n"); + stdin.flush(); + final String output = readUntilPrompt(); + if (errorPromptSeen) { + throw new GremlinExecutionException(statement, lastErrorText); + } + return output; + } + + @Override + public void close() { + running = false; + errorDismisser.interrupt(); + stdoutReaderThread.interrupt(); + process.destroyForcibly(); + try { + process.waitFor(5, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + closeQuietly(stdin); + closeQuietly(stdout); + closeQuietly(stderr); + } + + /** + * Background thread that reads stdout and feeds characters into the blocking queue. + */ + private void readStdout() { + try { + while (running) { + final int ch = stdout.read(); + stdoutQueue.put(ch); + if (ch == EOF) return; + } + } catch (final IOException | InterruptedException e) { + // Process closed or interrupted, signal EOF + try { + stdoutQueue.put(EOF); + } catch (final InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Reads from the stdout queue character-by-character until the prompt is detected. + */ + private String readUntilPrompt() throws IOException, ConsoleTimeoutException { + final StringBuilder buffer = new StringBuilder(); + final long deadline = System.currentTimeMillis() + timeoutMs; + + while (true) { + final long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + throw new ConsoleTimeoutException( + "Timed out after " + timeoutMs + "ms waiting for gremlin> prompt. Buffer contents:\n" + buffer); + } + + final Integer ch; + try { + ch = stdoutQueue.poll(Math.min(remaining, 100), TimeUnit.MILLISECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while reading console output", e); + } + + if (ch == null) continue; + + if (ch == EOF) { + throw new IOException("Console process ended unexpectedly. Buffer contents:\n" + buffer); + } + final char c = (char) ch.intValue(); + // Skip ANSI escape sequences (ESC [ ... letter) + if (c == '\u001B') { + inEscape = true; + continue; + } + if (inEscape) { + if (Character.isLetter(c)) { + inEscape = false; + } + continue; + } + buffer.append(c); + + if (buffer.length() >= PROMPT.length()) { + final String tail = buffer.substring(buffer.length() - PROMPT.length()); + if (tail.equals(PROMPT)) { + final String output = buffer.substring(0, buffer.length() - PROMPT.length()); + return output.trim(); + } + } + } + } + + /** + * Background thread that monitors stderr and automatically dismisses error prompts. + * The {@code Display stack trace? [yN]} prompt is the signal that the current statement + * raised a Gremlin error; the prompt is answered to unblock the process and the error is + * recorded so {@link #execute(String)} can surface it as a {@link GremlinExecutionException}. + * Error report text precedes the prompt on stderr, so it is captured for the message. + */ + private void dismissErrorPrompts() { + final int windowSize = ERROR_PROMPT.length(); + final StringBuilder errBuffer = new StringBuilder(); + try { + while (running) { + synchronized (stderrStdinLock) { + if (stderr.ready()) { + final int ch = stderr.read(); + if (ch == -1) return; + errBuffer.append((char) ch); + if (errorCapture.length() < MAX_ERROR_CAPTURE) { + errorCapture.append((char) ch); + } + + if (errBuffer.toString().contains(ERROR_PROMPT)) { + stdin.write("\n"); + stdin.flush(); + lastErrorText = errorCapture.toString().trim(); + errorPromptSeen = true; + errBuffer.setLength(0); + errorCapture.setLength(0); + } else if (errBuffer.length() > windowSize) { + errBuffer.delete(0, errBuffer.length() - windowSize); + } + } + } + Thread.sleep(50); + } + } catch (final InterruptedException e) { + // Expected on shutdown + } catch (final IOException e) { + // Process closed, exit silently + } + } + + private static void closeQuietly(final Closeable closeable) { + try { + if (closeable != null) closeable.close(); + } catch (final IOException ignored) { + // ignore + } + } + + /** + * Thrown when the console does not produce a prompt within the timeout period. + */ + public static class ConsoleTimeoutException extends Exception { + public ConsoleTimeoutException(final String message) { + super(message); + } + } + + /** + * @deprecated Use {@link ConsoleTimeoutException} instead. + */ + @Deprecated + public static class TimeoutException extends ConsoleTimeoutException { + public TimeoutException(final String message) { + super(message); + } + } + + /** + * Thrown when an executed statement produced a Gremlin error (signalled by the console's + * {@code Display stack trace? [yN]} prompt). Such errors must fail the docs build rather + * than render as silently-empty output. + */ + public static class GremlinExecutionException extends RuntimeException { + public GremlinExecutionException(final String statement, final String errorText) { + super("Gremlin statement failed: " + statement + + (errorText == null || errorText.isEmpty() ? "" : "\n" + errorText)); + } + } +} diff --git a/docs/preprocessor/awk/progressbar.groovy.template b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java similarity index 54% rename from docs/preprocessor/awk/progressbar.groovy.template rename to tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java index 7fcafea8e9c..a6e2e1400fd 100644 --- a/docs/preprocessor/awk/progressbar.groovy.template +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java @@ -16,20 +16,22 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.jruby.extension.spi.ExtensionRegistry; /** - * @author Daniel Kuppitz (http://gremlin.guru) + * SPI entry point that registers the TinkerPop documentation extensions with AsciidoctorJ. + * Registers both the Treeprocessor (for gremlin block processing) and the Postprocessor + * (for callout fixes, version replacement, etc.) */ -pb = { def progress -> - def barLength = 100 - def ratio = barLength / 100 - def builder = new StringBuilder() - def percent = (int) ((progress / TOTAL_LINES) * 100) - def progressLength = (int) ((progress / TOTAL_LINES) * (100 * ratio)) - builder.append('=' * progressLength) - if (progressLength < barLength) { - builder.append('>') - builder.append(' ' * (barLength - progressLength - 1)) - } - System.err.print(String.format("\r progress: [%s] %s", builder, "${percent}%")) +public class GremlinDocsExtension implements ExtensionRegistry { + + @Override + public void register(final Asciidoctor asciidoctor) { + asciidoctor.javaExtensionRegistry() + .treeprocessor(GremlinTreeprocessor.class) + .postprocessor(GremlinPostprocessor.class); + } } diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java new file mode 100644 index 00000000000..1f4fad4b8b0 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessor.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Document; +import org.asciidoctor.extension.Postprocessor; + +import java.util.regex.Pattern; + +/** + * Postprocessor that applies callout fixes, removes empty comment spans, + * and replaces x.y.z version placeholders with the actual TinkerPop version. + */ +public class GremlinPostprocessor extends Postprocessor { + + // Matches or (with possible other classes) + private static final Pattern CONUM_PATTERN = Pattern.compile( + "<([ib])\\s+class=\"conum\""); + + // Matches // preceding a callout marker, wraps in hide-when-copy span + private static final Pattern COMMENT_BEFORE_CONUM_PATTERN = Pattern.compile( + "//\\s*(<[ib] class=\"conum)"); + + // Matches empty comment spans from CodeRay: /* */ + private static final Pattern EMPTY_COMMENT_SPAN_PATTERN = Pattern.compile( + "/\\*\\s*\\*/"); + + @Override + public String process(final Document document, final String output) { + String result = output; + + // 1. Remove empty comment spans from CodeRay + result = EMPTY_COMMENT_SPAN_PATTERN.matcher(result).replaceAll(""); + + // 2. Replace x.y.z with actual version + final String version = resolveVersion(document); + if (version != null) { + result = result.replace("x.y.z", version); + } + + return result; + } + + private String resolveVersion(final Document document) { + Object version = document.getAttribute("tinkerpop-version"); + if (version != null) { + return version.toString(); + } + version = document.getAttribute("revnumber"); + if (version != null) { + return version.toString(); + } + return null; + } +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java new file mode 100644 index 00000000000..8bc60dae87f --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessor.java @@ -0,0 +1,745 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Block; +import org.asciidoctor.ast.Document; +import org.asciidoctor.ast.StructuralNode; +import org.asciidoctor.extension.Treeprocessor; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Walks the AsciiDoc AST after parsing and finds listing blocks with style {@code gremlin-groovy}. + * Executes the Gremlin code against a {@link GremlinConsole} and replaces block content with + * formatted console output. + */ +public class GremlinTreeprocessor extends Treeprocessor { + + private static final Logger LOG = Logger.getLogger(GremlinTreeprocessor.class.getName()); + + private static final String STYLE = "gremlin-groovy"; + private static final String PROMPT = "gremlin> "; + private static final String EXISTING = "existing"; + private static final String TAB_ATTR = "tab"; + static final String PLUGINS_EXCLUDE_ATTR = "gremlin-docs-plugins-exclude"; + + static final Set SUPPORTED_LANGUAGES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("groovy", "java", "csharp", "javascript", "python", "go"))); + + static final Map GRAPH_INIT; + + static { + final Map m = new HashMap<>(); + m.put("modern", "graph = TinkerFactory.createModern()"); + m.put("classic", "graph = TinkerFactory.createClassic()"); + m.put("crew", "graph = TinkerFactory.createTheCrew()"); + m.put("theCrew", "graph = TinkerFactory.createTheCrew()"); + m.put("grateful", "graph = TinkerFactory.createGratefulDead()"); + m.put("sink", "graph = TinkerFactory.createKitchenSink()"); + GRAPH_INIT = Collections.unmodifiableMap(m); + } + + private int gremlinBlockCount; + private final StatementExecutor executor; + private final TabbedHtmlBuilder tabBuilder; + private final ConsoleRestartHandler restartHandler; + private ConsoleRestartHandler activeRestartHandler; + private String currentGraph; + private List currentExcludedPlugins; + + /** + * Creates a GremlinTreeprocessor that processes blocks without executing them (dry-run mode). + */ + public GremlinTreeprocessor() { + this((StatementExecutor) null, null); + } + + /** + * Creates a GremlinTreeprocessor that executes blocks against the provided console. + * + * @param console the GremlinConsole to execute statements against, or null for dry-run + */ + public GremlinTreeprocessor(final GremlinConsole console) { + this(console == null ? null : statement -> console.execute(statement), null); + } + + /** + * Creates a GremlinTreeprocessor with a custom statement executor for testing. + * + * @param executor the executor to use, or null for dry-run + */ + GremlinTreeprocessor(final StatementExecutor executor) { + this(executor, null); + } + + /** + * Creates a GremlinTreeprocessor with a custom statement executor and restart handler. + * + * @param executor the executor to use, or null for dry-run + * @param restartHandler the handler invoked when plugin exclusions change, or null to ignore + */ + GremlinTreeprocessor(final StatementExecutor executor, final ConsoleRestartHandler restartHandler) { + this.executor = executor; + this.tabBuilder = new TabbedHtmlBuilder(); + this.restartHandler = restartHandler; + } + + private StatementExecutor resolvedExecutor; + private GremlinConsole lazyConsole; + private Path consoleHomePath; + private boolean dryRun; + private boolean sugarLoaded; + + @Override + public Document process(final Document document) { + gremlinBlockCount = 0; + currentGraph = null; + final Object dryRunAttr = document.getAttribute("gremlin-docs-dryrun"); + dryRun = dryRunAttr != null && !"false".equals(dryRunAttr.toString()); + + // Store console home for lazy init on first gremlin block + if (!dryRun && executor == null) { + final Object consoleHome = document.getAttribute("gremlin-docs-console-home"); + if (consoleHome != null && !consoleHome.toString().isEmpty()) { + consoleHomePath = Paths.get(consoleHome.toString()); + } else { + LOG.info("No gremlin-docs-console-home attribute; skipping console execution"); + } + } + + // Use an injected handler (tests) when present, otherwise default to physically toggling + // plugin directories under the resolved console home (production via SPI). + activeRestartHandler = restartHandler != null ? restartHandler + : (consoleHomePath != null ? new PluginDirectoryRestartHandler(consoleHomePath) : null); + + try { + checkPluginExclusions(document); + processBlock(document, dryRun); + LOG.info("Processed " + gremlinBlockCount + " gremlin blocks"); + } finally { + if (lazyConsole != null) { + lazyConsole.close(); + lazyConsole = null; + resolvedExecutor = null; + } + } + return document; + } + + /** + * Starts the console on demand when the first gremlin block is encountered. + */ + private void ensureConsoleStarted() { + if (resolvedExecutor != null || executor != null) return; + if (consoleHomePath == null) return; + try { + LOG.info("Starting GremlinConsole from: " + consoleHomePath); + lazyConsole = new GremlinConsole(consoleHomePath); + resolvedExecutor = statement -> lazyConsole.execute(statement); + // Match the old preprocessor: raise the console's result display limit so traversals + // returning many results (e.g. all-pairs shortestPath) render fully instead of being + // truncated with "..." at the interactive default. + lazyConsole.execute(":set max-iteration 100"); + LOG.info("GremlinConsole started successfully"); + } catch (final IOException | GremlinConsole.ConsoleTimeoutException e) { + LOG.warning("Failed to start GremlinConsole: " + e.getMessage()); + throw new ConsoleRestartedException("Console startup failed: " + e.getMessage()); + } + } + + /** + * Returns the number of gremlin-groovy listing blocks found during the last {@link #process} call. + */ + public int getGremlinBlockCount() { + return gremlinBlockCount; + } + + /** + * Establishes the document-level baseline exclusion set before the AST walk. An absent + * attribute means "no exclusions", so each book starts with every plugin enabled and any + * exclusion latched from a prior document is cleared. + */ + private void checkPluginExclusions(final Document document) { + if (activeRestartHandler == null) return; + final Object attrValue = document.hasAttribute(PLUGINS_EXCLUDE_ATTR) + ? document.getAttribute(PLUGINS_EXCLUDE_ATTR) : null; + applyExclusion(parseExcludeList(attrValue == null ? "" : attrValue.toString())); + } + + /** + * Applies a section-level {@code :gremlin-docs-plugins-exclude:} attribute encountered during + * the walk. Unlike the document baseline, an absent attribute on a section means "inherit" + * (no change), so only sections that declare the attribute trigger a transition. + */ + private void maybeApplySectionExclusion(final StructuralNode node) { + if (activeRestartHandler == null || !"section".equals(node.getContext())) return; + final Object exclude = node.getAttribute(PLUGINS_EXCLUDE_ATTR); + if (exclude != null) { + applyExclusion(parseExcludeList(exclude.toString())); + } + } + + /** + * Transitions the active plugin exclusion set. When it changes, the current console is closed, + * the restart handler toggles the plugin directories (and {@code plugins.txt}), and the next + * gremlin block lazily starts a fresh console with the new classpath. + */ + private void applyExclusion(final List excludeList) { + final List current = currentExcludedPlugins == null + ? Collections.emptyList() : currentExcludedPlugins; + currentExcludedPlugins = excludeList; + if (excludeList.equals(current)) return; + closeConsole(); + invokeRestartHandler(excludeList); + sugarLoaded = false; + } + + /** + * Parses a comma-separated list of plugin names into a sorted, deduplicated list. + */ + static List parseExcludeList(final String value) { + if (value == null || value.trim().isEmpty()) { + return Collections.emptyList(); + } + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .sorted() + .distinct() + .collect(Collectors.toList()); + } + + private void invokeRestartHandler(final List excludedPlugins) { + try { + activeRestartHandler.onRestart(excludedPlugins); + } catch (final IOException e) { + throw new RuntimeException("Failed to restart console with excluded plugins: " + excludedPlugins, e); + } + } + + private void processBlock(final StructuralNode node, final boolean dryRun) { + maybeApplySectionExclusion(node); + final List blocks = node.getBlocks(); + for (int i = 0; i < blocks.size(); i++) { + final StructuralNode block = blocks.get(i); + if ("listing".equals(block.getContext()) && STYLE.equals(block.getStyle())) { + gremlinBlockCount++; + LOG.info("Processing gremlin block #" + gremlinBlockCount); + i = processGremlinTabGroup(node, i, dryRun); + } else if (isStandaloneTabBlock(block)) { + i = processStandaloneTabGroup(node, i); + } else { + processBlock(block, dryRun); + } + } + } + + /** + * Processes a gremlin-groovy block and any consecutive manual language variant siblings + * into a tabbed HTML group. Returns the index to continue iteration from. + */ + private int processGremlinTabGroup(final StructuralNode parent, final int startIndex, final boolean dryRun) { + final List blocks = parent.getBlocks(); + final Block gremlinBlock = (Block) blocks.get(startIndex); + + final String consoleOutput = buildConsoleOutput(gremlinBlock, dryRun); + final List tabs = new ArrayList<>(); + tabs.add(TabbedHtmlBuilder.consoleTabHighlighted("groovy", + highlightAsGroovy(parent, consoleOutput))); + // Add second tab with clean source code (no prompts/output) + tabs.add(TabbedHtmlBuilder.codeTabHighlighted("groovy", + highlightAsGroovy(parent, gremlinBlock.getSource()))); + + // Consume consecutive [source,] sibling blocks as manual tabs (FR-5) + int lastIndex = startIndex; + for (int j = startIndex + 1; j < blocks.size(); j++) { + final StructuralNode sibling = blocks.get(j); + if (isManualTabBlock(sibling)) { + final Block sourceBlock = (Block) sibling; + final String lang = getSourceLanguage(sourceBlock); + tabs.add(TabbedHtmlBuilder.codeTabHighlighted(lang, + highlightAsSource(parent, lang, sourceBlock.getSource()))); + lastIndex = j; + } else { + break; + } + } + + final String html = tabBuilder.build(tabs); + replaceWithPassBlock(parent, startIndex, lastIndex, html); + return startIndex; + } + + /** + * Highlights source code using CodeRay via the JRuby runtime bundled with AsciidoctorJ. + */ + private String highlightAsGroovy(final StructuralNode parent, final String source) { + return highlightAsSource(parent, "groovy", source); + } + + private String highlightAsSource(final StructuralNode parent, final String lang, final String source) { + if (source == null || source.isEmpty()) return ""; + // Strip callouts before highlighting, re-inject after + final String[] lines = source.split("\\r?\\n"); + final String[] calloutMarkers = new String[lines.length]; + final StringBuilder cleanSource = new StringBuilder(); + for (int i = 0; i < lines.length; i++) { + final java.util.regex.Matcher m = java.util.regex.Pattern + .compile("\\s*((<\\d+>\\s*)+)$").matcher(lines[i]); + if (m.find()) { + calloutMarkers[i] = m.group(1); + if (i > 0) cleanSource.append("\n"); + cleanSource.append(lines[i], 0, m.start()); + } else { + calloutMarkers[i] = null; + if (i > 0) cleanSource.append("\n"); + cleanSource.append(lines[i]); + } + } + + String highlighted = doHighlight(parent, lang, cleanSource.toString()); + + // Re-inject callouts as HTML conums + final String[] highlightedLines = highlighted.split("\\n", -1); + final StringBuilder result = new StringBuilder(); + for (int i = 0; i < highlightedLines.length; i++) { + if (i > 0) result.append("\n"); + result.append(highlightedLines[i]); + if (i < calloutMarkers.length && calloutMarkers[i] != null) { + final java.util.regex.Matcher nums = java.util.regex.Pattern + .compile("<(\\d+)>").matcher(calloutMarkers[i]); + while (nums.find()) { + result.append(" // (") + .append(nums.group(1)).append(")"); + } + } + } + return result.toString(); + } + + private org.jruby.runtime.builtin.IRubyObject coderayEncoder; + private org.jruby.Ruby rubyRuntime; + + private String doHighlight(final StructuralNode parent, final String lang, final String source) { + try { + if (rubyRuntime == null) { + rubyRuntime = org.asciidoctor.jruby.internal.JRubyRuntimeContext.get(parent); + if (rubyRuntime == null) return escapeHtml(source); + coderayEncoder = rubyRuntime.evalScriptlet( + "require 'coderay'; CodeRay::Duo[:groovy, :html, :css => :class]"); + } + final org.jruby.RubyString rubySource = org.jruby.RubyString.newString(rubyRuntime, source); + final org.jruby.runtime.builtin.IRubyObject result = coderayEncoder.callMethod( + rubyRuntime.getCurrentContext(), "highlight", rubySource); + return result != null ? result.asJavaString() : escapeHtml(source); + } catch (final Exception e) { + LOG.warning("CodeRay highlighting failed, falling back to plain: " + e.getMessage()); + return escapeHtml(source); + } + } + + private static String escapeHtml(final String text) { + if (text == null) return ""; + return text.replace("&", "&").replace("<", "<") + .replace(">", ">").replace("\"", """); + } + + /** + * Processes consecutive standalone [source,,tab] blocks into a tab group (FR-7). + * Returns the index to continue iteration from. + */ + private int processStandaloneTabGroup(final StructuralNode parent, final int startIndex) { + final List blocks = parent.getBlocks(); + final List tabs = new ArrayList<>(); + + int lastIndex = startIndex; + for (int j = startIndex; j < blocks.size(); j++) { + final StructuralNode block = blocks.get(j); + if (isStandaloneTabBlock(block) || isManualTabBlock(block)) { + final Block sourceBlock = (Block) block; + final String lang = getSourceLanguage(sourceBlock); + tabs.add(TabbedHtmlBuilder.codeTabHighlighted(lang, + highlightAsSource(parent, lang, sourceBlock.getSource()))); + lastIndex = j; + } else { + break; + } + } + + final String html = tabBuilder.build(tabs); + replaceWithPassBlock(parent, startIndex, lastIndex, html); + return startIndex; + } + + /** + * Checks if a block is a [source,] listing block with a supported language (not tab-annotated). + */ + private static boolean isManualTabBlock(final StructuralNode block) { + if (!"listing".equals(block.getContext())) return false; + if (!"source".equals(block.getStyle())) return false; + final Object thirdAttr = block.getAttributes().get("3"); + if (thirdAttr != null && TAB_ATTR.equals(thirdAttr.toString().trim())) return false; + final String lang = getSourceLanguage((Block) block); + return lang != null && SUPPORTED_LANGUAGES.contains(lang); + } + + /** + * Checks if a block is a standalone tab block: [source,,tab]. + */ + private static boolean isStandaloneTabBlock(final StructuralNode block) { + if (!"listing".equals(block.getContext())) return false; + if (!"source".equals(block.getStyle())) return false; + final Object thirdAttr = block.getAttributes().get("3"); + if (thirdAttr == null) return false; + if (!TAB_ATTR.equals(thirdAttr.toString().trim())) return false; + final String lang = getSourceLanguage((Block) block); + return lang != null && SUPPORTED_LANGUAGES.contains(lang); + } + + /** + * Gets the source language from a listing block's attributes. + */ + private static String getSourceLanguage(final Block block) { + final Object langAttr = block.getAttributes().get("language"); + if (langAttr != null) { + final String lang = langAttr.toString().trim(); + if (!lang.isEmpty()) return lang; + } + final Object attr = block.getAttributes().get("2"); + if (attr == null) return null; + final String lang = attr.toString().trim(); + return lang.isEmpty() ? null : lang; + } + + /** + * Replaces blocks from startIndex to endIndex (inclusive) with a pass block containing raw HTML. + */ + private void replaceWithPassBlock(final StructuralNode parent, final int startIndex, final int endIndex, + final String html) { + final List blocks = parent.getBlocks(); + for (int j = endIndex; j >= startIndex; j--) { + blocks.remove(j); + } + final Map attrs = new HashMap<>(); + final Block passBlock = createBlock(parent, "pass", html, attrs); + blocks.add(startIndex, passBlock); + } + + private String buildConsoleOutput(final Block block, final boolean dryRun) { + try { + return doBuildConsoleOutput(block, dryRun); + } catch (final ConsoleRestartedException e) { + // Console was restarted — retry the entire block once on the fresh console + LOG.info("Retrying block after console restart"); + return doBuildConsoleOutput(block, dryRun); + } + } + + private String doBuildConsoleOutput(final Block block, final boolean dryRun) { + if (!dryRun) { + ensureConsoleStarted(); + } + + final String source = block.getSource(); + if (source == null || source.isEmpty()) { + return ""; + } + + final String[] lines = source.split("\\r?\\n"); + final List displayStatements = buildDisplayStatements(lines); + final List execStatements = buildStatements(lines); + + // Sugar syntax permanently mutates the Groovy metaclass and only takes effect when + // SugarLoader.load() runs on a pristine metaclass -- i.e. BEFORE any traversal has been + // used in the JVM. Since the console is long-lived and has executed prior blocks, always + // restart to get a fresh console, then load sugar before initializing the graph/source. + final boolean needsSugar = !dryRun && execStatements.stream().anyMatch(GremlinTreeprocessor::usesSugarSyntax); + if (needsSugar && getActiveExecutor() != null) { + restartConsole(); + executeSafely("org.apache.tinkerpop.gremlin.groovy.loaders.SugarLoader.load()"); + sugarLoaded = true; + final String graphName = extractGraphName(block); + initGraphIfNeeded(graphName); + } else if (!dryRun && getActiveExecutor() != null) { + if (sugarLoaded) { + // Previous block loaded sugar; restart to get a clean metaclass. + restartConsole(); + sugarLoaded = false; + } + final String graphName = extractGraphName(block); + initGraphIfNeeded(graphName); + } + + final StringBuilder output = new StringBuilder(); + for (int s = 0; s < displayStatements.size(); s++) { + // Show original lines with callouts for display + final String[] displayLines = displayStatements.get(s).split("\\r?\\n"); + for (int l = 0; l < displayLines.length; l++) { + if (l == 0) { + output.append(PROMPT).append(displayLines[l]).append("\n"); + } else { + // Continuation lines: indent to align with first line (no prompt) + output.append(" ").append(displayLines[l]).append("\n"); + } + } + if (!dryRun && getActiveExecutor() != null) { + final String result = executeSafely(execStatements.get(s)); + if (result != null && !result.isEmpty()) { + final String[] resultLines = result.split("\\r?\\n"); + // Skip echo lines (first line + continuation prompts like ......N>) + for (int idx = 1; idx < resultLines.length; idx++) { + if (resultLines[idx].matches("^\\.{6}\\d+>.*")) continue; + output.append(resultLines[idx]).append("\n"); + } + } + } + } + return output.toString().stripTrailing(); + } + + /** + * Groups source lines into complete statements for execution. Strips callouts. + */ + static List buildStatements(final String[] lines) { + return groupStatements(lines, true); + } + + private static int countOccurrences(final String str, final String sub) { + int count = 0; + int idx = 0; + while ((idx = str.indexOf(sub, idx)) != -1) { count++; idx += sub.length(); } + return count; + } + + /** + * Groups source lines into complete statements for display. Preserves callouts. + */ + static List buildDisplayStatements(final String[] lines) { + return groupStatements(lines, false); + } + + /** + * Groups source lines into complete statements. A new statement starts only at a + * non-indented line when the accumulated statement has balanced brackets and is not + * inside a triple-quoted string. Tracking bracket depth keeps multi-line Groovy + * constructs (e.g. {@code (1..10).each \{ ... \}}) together even though their closing + * line is not indented, which would otherwise send an incomplete statement to the + * console and hang at a continuation prompt. + * + * @param strip when {@code true}, callouts are stripped from the emitted text (for + * execution); when {@code false}, the original line is emitted (for display) + */ + private static List groupStatements(final String[] lines, final boolean strip) { + final List statements = new ArrayList<>(); + final StringBuilder current = new StringBuilder(); + boolean inTripleQuote = false; + int depth = 0; + for (final String line : lines) { + final String detect = stripCallouts(line); + final String content = strip ? detect : line; + final int tqCount = countOccurrences(detect, "\"\"\""); + final boolean touchesTriple = inTripleQuote || tqCount > 0; + if (current.length() == 0) { + current.append(content); + } else if (inTripleQuote || depth > 0 + || (detect.length() > 0 && Character.isWhitespace(detect.charAt(0)))) { + current.append("\n").append(content); + } else { + statements.add(current.toString()); + current.setLength(0); + depth = 0; + current.append(content); + } + if (!touchesTriple) depth += bracketDelta(detect); + if (tqCount % 2 != 0) inTripleQuote = !inTripleQuote; + } + if (current.length() > 0) { + statements.add(current.toString()); + } + return statements; + } + + /** + * Net change in bracket nesting ({@code (} {@code [} {@code \{}) contributed by a line, + * ignoring brackets inside single- or double-quoted string literals (so GString + * interpolation like {@code ${it}} does not affect the count). + */ + private static int bracketDelta(final String s) { + int d = 0; + boolean sq = false; + boolean dq = false; + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (sq) { + if (c == '\\') i++; + else if (c == '\'') sq = false; + } else if (dq) { + if (c == '\\') i++; + else if (c == '"') dq = false; + } else if (c == '\'') { + sq = true; + } else if (c == '"') { + dq = true; + } else if (c == '{' || c == '(' || c == '[') { + d++; + } else if (c == '}' || c == ')' || c == ']') { + d--; + } + } + return d; + } + + /** + * Strips AsciiDoc callout markers (e.g. {@code <1>}, {@code <2>}) from the end of a line. + */ + static String stripCallouts(final String line) { + return line.replaceAll("\\s*((<\\d+>\\s*)*<\\d+>)\\s*$", ""); + } + + /** + * Detects sugar-syntax usage that requires SugarLoader: bare step properties + * (e.g. {@code g.V}, {@code g.V.name}) or range operators (e.g. {@code g.V[0..2]}). + */ + static boolean usesSugarSyntax(final String statement) { + if (statement == null) return false; + // Range access: g.V[...] or g.E[...] + if (statement.matches(".*\\bg\\.[VE]\\s*\\[.*")) return true; + // Bare step property: g.V or g.E not immediately followed by '(' + final java.util.regex.Matcher m = java.util.regex.Pattern + .compile("\\bg\\.[VE]([^(\\w]|$)").matcher(statement); + return m.find(); + } + + /** + * Extracts the graph name from the second positional attribute of the block. + */ + static String extractGraphName(final StructuralNode block) { + final Object attr = block.getAttributes().get("2"); + if (attr == null) { + return null; + } + final String name = attr.toString().trim(); + return name.isEmpty() ? null : name; + } + + /** + * Initializes the graph and traversal source for the block. Re-initializes for + * every non-{@code existing} block because prior blocks may have mutated + * {@code graph} or reassigned {@code g} (e.g. {@code g = ...withComputer()}), + * which would otherwise leak into subsequent blocks that expect a fresh source. + */ + private void initGraphIfNeeded(final String graphName) { + if (EXISTING.equals(graphName)) { + return; + } + + // Clear file-backed graph stores reused across blocks. Neo4j holds an exclusive + // store lock, so a stale '/tmp/neo4j' left by a prior block (or prior build run) + // makes the next Neo4jGraph.open('/tmp/neo4j') hang acquiring the lock. Close any + // prior graph to release its lock, then delete the dirs -- mirroring the old + // preprocessor which cleared these before every graph-init block. + executeSafely("try { if (binding.hasVariable('graph') && graph != null) graph.close() } catch (e) {}"); + executeSafely("['/tmp/neo4j', '/tmp/tinkergraph.kryo'].each { p -> " + + "def f = new File(p); if (f.exists()) f.deleteDir() }"); + + final String initStatement; + if (graphName == null) { + initStatement = "graph = TinkerGraph.open()"; + } else { + initStatement = GRAPH_INIT.getOrDefault(graphName, "graph = TinkerGraph.open()"); + } + + executeSafely(initStatement); + executeSafely("g = graph.traversal()"); + currentGraph = graphName; + } + + private StatementExecutor getActiveExecutor() { + return executor != null ? executor : resolvedExecutor; + } + + private String executeSafely(final String statement) { + try { + return getActiveExecutor().execute(statement); + } catch (final GremlinConsole.GremlinExecutionException e) { + // A genuine Gremlin error must fail the build, not render as empty output. + throw e; + } catch (final GremlinConsole.ConsoleTimeoutException | IOException e) { + // Console may have died — restart it and propagate to retry at block level + LOG.warning("Console appears dead, restarting: " + e.getMessage()); + restartConsole(); + throw new ConsoleRestartedException("Console restarted due to: " + e.getMessage()); + } catch (final Exception e) { + throw new RuntimeException("Failed to execute statement: " + statement, e); + } + } + + private void restartConsole() { + closeConsole(); + if (lazyConsoleWasClosed) { + // Allow OS to reclaim resources from dead console and its children + try { Thread.sleep(2000); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); } + } + currentGraph = null; + ensureConsoleStarted(); + } + + private boolean lazyConsoleWasClosed; + + /** Closes the lazily-started console (if any) so the next block starts a fresh one. No-op in test mode. */ + private void closeConsole() { + lazyConsoleWasClosed = lazyConsole != null; + if (lazyConsole != null) { + lazyConsole.close(); + lazyConsole = null; + resolvedExecutor = null; + } + currentGraph = null; + } + + /** + * Thrown when the console was restarted mid-block to signal that the block should be retried. + */ + static class ConsoleRestartedException extends RuntimeException { + ConsoleRestartedException(final String message) { + super(message); + } + } + + /** + * Functional interface for executing Gremlin statements, enabling testability. + */ + @FunctionalInterface + interface StatementExecutor { + String execute(String statement) throws IOException, GremlinConsole.ConsoleTimeoutException; + } +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandler.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandler.java new file mode 100644 index 00000000000..078cf193b06 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandler.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Physically toggles plugin availability in a Gremlin Console distribution so that conflicting + * plugins (e.g. Neo4j's Scala 2.11 vs Spark's Scala 2.12) never share one classpath. + *

+ * The console's {@code bin/gremlin.sh} composes its classpath from {@code lib/*.jar} plus + * {@code ext//plugin/*}. To exclude a plugin this handler moves {@code ext/} + * out to {@code ext-disabled/} (off-classpath) and removes its activation class from + * {@code ext/plugins.txt}; to re-include it the directory is moved back and the class restored. + * Keeping {@code plugins.txt} in sync is required because the console rewrites that file on + * startup, permanently dropping any listed plugin whose jars are missing. + */ +final class PluginDirectoryRestartHandler implements ConsoleRestartHandler { + + private static final Logger LOG = Logger.getLogger(PluginDirectoryRestartHandler.class.getName()); + + /** Toggleable plugin directory -> activation class written to ext/plugins.txt. */ + private static final Map TOGGLEABLE = Collections.unmodifiableMap(new LinkedHashMap() {{ + put("neo4j-gremlin", "org.apache.tinkerpop.gremlin.neo4j.jsr223.Neo4jGremlinPlugin"); + put("spark-gremlin", "org.apache.tinkerpop.gremlin.spark.jsr223.SparkGremlinPlugin"); + put("hadoop-gremlin", "org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopGremlinPlugin"); + }}); + + private final Path extDir; + private final Path disabledDir; + private final Path pluginsTxt; + + PluginDirectoryRestartHandler(final Path consoleHome) { + this.extDir = consoleHome.resolve("ext"); + this.disabledDir = consoleHome.resolve("ext-disabled"); + this.pluginsTxt = extDir.resolve("plugins.txt"); + } + + @Override + public void onRestart(final List excludedPlugins) throws IOException { + for (final String plugin : TOGGLEABLE.keySet()) { + if (excludedPlugins.contains(plugin)) { + disable(plugin); + } else { + enable(plugin); + } + } + } + + private void disable(final String plugin) throws IOException { + final Path active = extDir.resolve(plugin); + final Path disabled = disabledDir.resolve(plugin); + if (Files.isDirectory(active)) { + // The active copy is authoritative. Clear any stale disabled copy left by an + // interrupted run before moving, since Files.move(REPLACE_EXISTING) cannot replace + // a non-empty directory. + Files.createDirectories(disabledDir); + deleteRecursively(disabled); + Files.move(active, disabled); + LOG.info("Excluded plugin: " + plugin); + } + setPluginEnabled(TOGGLEABLE.get(plugin), false); + } + + private void enable(final String plugin) throws IOException { + final Path active = extDir.resolve(plugin); + final Path disabled = disabledDir.resolve(plugin); + if (Files.isDirectory(disabled)) { + if (Files.isDirectory(active)) { + // Plugin is already present in ext/ (e.g. freshly installed); the active copy + // wins. Just drop the leftover disabled duplicate. + deleteRecursively(disabled); + } else { + Files.move(disabled, active); + LOG.info("Restored plugin: " + plugin); + } + } + setPluginEnabled(TOGGLEABLE.get(plugin), true); + } + + /** Recursively deletes a directory tree if it exists; a no-op otherwise. */ + private static void deleteRecursively(final Path path) throws IOException { + if (!Files.exists(path)) return; + try (java.util.stream.Stream walk = Files.walk(path)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (final IOException e) { + throw new java.io.UncheckedIOException(e); + } + }); + } catch (final java.io.UncheckedIOException e) { + throw e.getCause(); + } + } + + /** Adds or removes a single activation class line in ext/plugins.txt, preserving the rest. */ + private void setPluginEnabled(final String pluginClass, final boolean enabled) throws IOException { + if (!Files.exists(pluginsTxt)) return; + final List lines = Files.readAllLines(pluginsTxt).stream() + .filter(l -> !l.trim().equals(pluginClass)) + .collect(Collectors.toList()); + if (enabled) lines.add(pluginClass); + Files.write(pluginsTxt, lines); + } +} diff --git a/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java new file mode 100644 index 00000000000..707f918c0b2 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilder.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Generates CSS-only tabbed HTML matching the existing {@code tinkerpop.css} tab styles. + *

+ * The generated structure uses radio inputs for tab switching without JavaScript: + * 

+ * <section class="tabs tabs-N">
+ *   <input type="radio" name="tab-group-ID" id="tab-ID-1" class="tab-selector-1" checked>
+ *   <label for="tab-ID-1" class="tab-label-1">Label</label>
+ *   ...
+ *   <div class="tabcontent">
+ *     <div class="tabcontent-1">...</div>
+ *     ...
+ *   </div>
+ * </section>
+ *
+ */ +public class TabbedHtmlBuilder { + + private static final Pattern CALLOUT_PATTERN = Pattern.compile("<(\\d+)>"); + + private int groupCounter = 0; + + /** + * A single tab entry with a label, language, and source code content. + */ + static class Tab { + private final String label; + private final String language; + private final String content; + private final boolean preHighlighted; + + Tab(final String label, final String language, final String content) { + this(label, language, content, false); + } + + Tab(final String label, final String language, final String content, final boolean preHighlighted) { + this.label = label; + this.language = language; + this.content = content; + this.preHighlighted = preHighlighted; + } + + String getLabel() { + return label; + } + + String getLanguage() { + return language; + } + + String getContent() { + return content; + } + + boolean isPreHighlighted() { + return preHighlighted; + } + } + + /** + * Returns the current group counter value (for testing). + */ + int getGroupCounter() { + return groupCounter; + } + + /** + * Resets the group counter (for testing or reprocessing). + */ + void resetCounter() { + groupCounter = 0; + } + + /** + * Builds tabbed HTML for a list of tabs. + * + * @param tabs the tabs to render + * @return the complete HTML string for the tab group + */ + String build(final List tabs) { + if (tabs == null || tabs.isEmpty()) { + return ""; + } + + final int groupId = ++groupCounter; + final int tabCount = tabs.size(); + final StringBuilder html = new StringBuilder(); + + html.append("
\n"); + + // Radio inputs and labels + for (int i = 0; i < tabCount; i++) { + final int tabNum = i + 1; + final String inputId = "tab-" + groupId + "-" + tabNum; + final String checked = (i == 0) ? " checked" : ""; + + html.append(" \n"); + html.append(" \n"); + } + + html.append("
\n"); + + // Tab content panels + for (int i = 0; i < tabCount; i++) { + final Tab tab = tabs.get(i); + final int tabNum = i + 1; + html.append("
\n"); + html.append("
\n
\n"); + html.append("
")
+                    .append(tab.isPreHighlighted() ? tab.getContent() : renderContent(tab.getContent()))
+                    .append("
\n"); + html.append("
\n
\n"); + html.append("
\n"); + } + + html.append("
\n"); + html.append("
"); + + return html.toString(); + } + + /** + * Creates a console tab for a gremlin-groovy block. + * + * @param lang the language (e.g., "groovy") + * @param consoleOutput the formatted console output + * @return a Tab instance for the console output + */ + static Tab consoleTab(final String lang, final String consoleOutput) { + return new Tab("console (" + lang + ")", lang, consoleOutput); + } + + /** + * Creates a console tab with pre-highlighted HTML content. + */ + static Tab consoleTabHighlighted(final String lang, final String highlightedHtml) { + return new Tab("console (" + lang + ")", lang, highlightedHtml, true); + } + + /** + * Creates a code tab with pre-highlighted HTML content. + */ + static Tab codeTabHighlighted(final String lang, final String highlightedHtml) { + return new Tab(lang, lang, highlightedHtml, true); + } + + /** + * Creates a code tab for a manual language variant block. + * + * @param lang the language identifier + * @param source the source code + * @return a Tab instance for the code + */ + static Tab codeTab(final String lang, final String source) { + return new Tab(lang, lang, transformCallouts(source)); + } + + /** + * Transforms AsciiDoc callout markers (e.g., {@code <1>}) into comment form ({@code // <1>}). + * + * @param source the source code potentially containing callout markers + * @return the source with callout markers transformed to comments + */ + static String transformCallouts(final String source) { + if (source == null || source.isEmpty()) { + return source; + } + + final StringBuilder result = new StringBuilder(); + final String[] lines = source.split("\\r?\\n"); + for (int i = 0; i < lines.length; i++) { + if (i > 0) { + result.append("\n"); + } + result.append(transformCalloutLine(lines[i])); + } + return result.toString(); + } + + /** + * Transforms callout markers at the end of a line into comment form. + */ + private static String transformCalloutLine(final String line) { + final Matcher matcher = CALLOUT_PATTERN.matcher(line); + final StringBuilder sb = new StringBuilder(); + int lastEnd = 0; + final List callouts = new ArrayList<>(); + + while (matcher.find()) { + // Only transform callouts that appear at the end of the line (possibly with whitespace) + final String afterMatch = line.substring(matcher.end()).trim(); + if (afterMatch.isEmpty() || CALLOUT_PATTERN.matcher(afterMatch).matches()) { + callouts.add(matcher.group(1)); + if (sb.length() == 0) { + sb.append(line, 0, matcher.start()); + } + } else { + // Not a trailing callout, keep as-is + return line; + } + lastEnd = matcher.end(); + } + + if (callouts.isEmpty()) { + return line; + } + + // Trim trailing whitespace before callouts and append as comments + final String codePart = sb.toString().stripTrailing(); + final StringBuilder result = new StringBuilder(codePart); + for (final String num : callouts) { + result.append(" // <").append(num).append(">"); + } + return result.toString(); + } + + private static String escapeHtml(final String text) { + if (text == null) return ""; + return text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """); + } + + /** + * Escapes HTML and then renders callout markers as proper conum elements. + * Converts escaped {@code <N>} patterns into HTML callout spans. + */ + private static String renderContent(final String text) { + final String escaped = escapeHtml(text); + // Process line by line to handle callouts at end of lines + final String[] lines = escaped.split("\\n", -1); + final StringBuilder result = new StringBuilder(); + for (int i = 0; i < lines.length; i++) { + if (i > 0) result.append("\n"); + result.append(renderCallouts(lines[i])); + } + return result.toString(); + } + + /** + * Replaces trailing callout markers on a line with HTML conum elements. + */ + private static String renderCallouts(final String line) { + // Match trailing callout markers: <1> or <1> <2> etc. + final java.util.regex.Matcher m = java.util.regex.Pattern + .compile("\\s*((<\\d+>\\s*)+)$").matcher(line); + if (!m.find()) return line; + final String prefix = line.substring(0, m.start()); + final String calloutsPart = m.group(1); + final StringBuilder sb = new StringBuilder(prefix); + final java.util.regex.Matcher nums = java.util.regex.Pattern + .compile("<(\\d+)>").matcher(calloutsPart); + while (nums.find()) { + sb.append(" // (") + .append(nums.group(1)).append(")"); + } + return sb.toString(); + } +} diff --git a/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry b/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry new file mode 100644 index 00000000000..6a81ac1f60e --- /dev/null +++ b/tools/tinkerpop-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry @@ -0,0 +1 @@ +org.apache.tinkerpop.gremlin.docs.GremlinDocsExtension diff --git a/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml new file mode 100644 index 00000000000..1155717eb62 --- /dev/null +++ b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/core-site.xml @@ -0,0 +1,33 @@ + + + + + + fs.defaultFS + file:/// + + diff --git a/docs/preprocessor/control-characters.sh b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties old mode 100755 new mode 100644 similarity index 71% rename from docs/preprocessor/control-characters.sh rename to tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties index b8336d6bf01..6ee38de2b7d --- a/docs/preprocessor/control-characters.sh +++ b/tools/tinkerpop-docs/src/main/resources/hadoop-conf/hadoop-docs.properties @@ -1,6 +1,3 @@ -#!/bin/bash -# -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -17,9 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -# The main purpose of this script is to remove control characters -# that are occasionally hidden in the Groovy console's output +# +# Hadoop/Spark properties for TinkerPop docs builds. +# No Hadoop daemons are required -- all I/O uses the local filesystem +# via fs.defaultFS=file:/// (see core-site.xml in this directory). +# -sed -r 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' +# Disable distributed cache since we run locally with all jars on the classpath. +gremlin.hadoop.jarsInDistributedCache=false diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java new file mode 100644 index 00000000000..d1a7fe5c869 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinConsoleTest.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link GremlinConsole} that verify prompt detection, timeout, and error dismissal + * without requiring a real Gremlin Console process. + */ +public class GremlinConsoleTest { + + @Test + public void shouldDetectPromptAndReturnOutput() throws Exception { + final String fullStdout = "gremlin>" + "==>v[1]\n==>v[2]\ngremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("g.V()"); + assertThat(result, is("==>v[1]\n==>v[2]")); + } finally { + console.close(); + } + } + + @Test + public void shouldReturnEmptyForPromptOnly() throws Exception { + final String fullStdout = "gremlin>" + "gremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("1+1"); + assertThat(result, is("")); + } finally { + console.close(); + } + } + + @Test + public void shouldHandleMultiLineOutput() throws Exception { + final String fullStdout = "gremlin>" + "line1\nline2\nline3\ngremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + + try { + final String result = console.execute("something"); + assertThat(result, is("line1\nline2\nline3")); + } finally { + console.close(); + } + } + + @Test + public void shouldTimeoutWhenNoPromptReceived() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + // Use 200ms timeout for fast test + final GremlinConsole console = new GremlinConsole(mockProcess, 200); + try { + console.execute("g.V()"); + throw new AssertionError("Expected ConsoleTimeoutException"); + } catch (final GremlinConsole.ConsoleTimeoutException e) { + assertThat(e.getMessage(), containsString("Timed out after 200ms")); + assertThat(e.getMessage(), containsString("Buffer contents:")); + } finally { + console.close(); + feeder.close(); + } + } + + @Test + public void shouldIncludeBufferContentsInTimeoutMessage() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds, then partial output + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.write("partial output here".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + final GremlinConsole console = new GremlinConsole(mockProcess, 200); + try { + console.execute("test"); + throw new AssertionError("Expected ConsoleTimeoutException"); + } catch (final GremlinConsole.ConsoleTimeoutException e) { + assertThat(e.getMessage(), containsString("partial output here")); + } finally { + console.close(); + feeder.close(); + } + } + + @Test + public void shouldDismissErrorPromptOnStderr() throws Exception { + // The async dismisser must answer the "Display stack trace? [yN]" prompt on stderr + // (sending a newline to stdin) so the console process is never left blocked. Surfacing + // the error to the caller is covered by shouldThrowExecutionExceptionWhenErrorPromptSeen. + final String fullStdout = "gremlin>"; + final String stderrContent = "Display stack trace? [yN]"; + final ByteArrayOutputStream capturedStdin = new ByteArrayOutputStream(); + final GremlinConsole console = createConsoleWithStdin(fullStdout, stderrContent, capturedStdin); + + try { + // Poll until the error dismisser has written to stdin, with timeout + final long deadline = System.currentTimeMillis() + 5000; + while (capturedStdin.size() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + // Verify that a newline was sent to dismiss the error prompt + final String sent = capturedStdin.toString(StandardCharsets.UTF_8.name()); + assertThat(sent.contains("\n"), is(true)); + } finally { + console.close(); + } + } + + @Test + public void shouldThrowExecutionExceptionWhenErrorPromptSeen() throws Exception { + // Mirror the real console ordering: after the statement is sent, the error report + + // "Display stack trace?" prompt arrive on stderr and the process blocks on stdin until + // answered, so the next gremlin> prompt on stdout appears only AFTER the dismisser + // replies. execute() must then surface a GremlinExecutionException so the build fails. + final PipedOutputStream stdoutFeeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(stdoutFeeder); + stdoutFeeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + stdoutFeeder.flush(); + + final PipedOutputStream stderrFeeder = new PipedOutputStream(); + final PipedInputStream stderrStream = new PipedInputStream(stderrFeeder); + final ByteArrayOutputStream capturedStdin = new ByteArrayOutputStream(); + final GremlinConsole console = new GremlinConsole( + new MockProcess(stdoutStream, stderrStream, capturedStdin), 5_000); + + final AtomicReference thrown = new AtomicReference<>(); + final Thread exec = new Thread(() -> { + try { + console.execute("g.V().fail('boom')"); + } catch (final Throwable t) { + thrown.set(t); + } + }); + exec.start(); + try { + // Emit the error report + prompt on stderr only after execute() has started, so the + // per-statement error flag is reset before the dismisser observes the prompt. + Thread.sleep(200); + stderrFeeder.write("fail() Step Triggered\nDisplay stack trace? [yN]".getBytes(StandardCharsets.UTF_8)); + stderrFeeder.flush(); + // Wait for the dismisser to answer (an extra newline beyond the statement itself, + // which execute() also wrote to stdin), then release the next prompt on stdout. + final int statementBytes = "g.V().fail('boom')\n".getBytes(StandardCharsets.UTF_8).length; + final long deadline = System.currentTimeMillis() + 5000; + while (capturedStdin.size() <= statementBytes && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + stdoutFeeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + stdoutFeeder.flush(); + exec.join(5000); + } finally { + console.close(); + stdoutFeeder.close(); + stderrFeeder.close(); + } + assertThat(thrown.get(), is(notNullValue())); + assertThat(thrown.get(), instanceOf(GremlinConsole.GremlinExecutionException.class)); + assertThat(thrown.get().getMessage(), containsString("g.V().fail('boom')")); + } + + @Test + public void shouldThrowIOExceptionWhenProcessDiesMidRead() throws Exception { + final PipedOutputStream feeder = new PipedOutputStream(); + final PipedInputStream stdoutStream = new PipedInputStream(feeder); + + // Write initial prompt so constructor succeeds + feeder.write("gremlin>".getBytes(StandardCharsets.UTF_8)); + feeder.write("partial".getBytes(StandardCharsets.UTF_8)); + feeder.flush(); + // Close the stream to simulate process death (read returns -1) + feeder.close(); + + final Process mockProcess = new MockProcess( + stdoutStream, + new ByteArrayInputStream(new byte[0]), + new ByteArrayOutputStream()); + + final GremlinConsole console = new GremlinConsole(mockProcess, 5000); + try { + console.execute("g.V()"); + throw new AssertionError("Expected IOException"); + } catch (final IOException e) { + assertThat(e.getMessage(), containsString("Console process ended unexpectedly")); + assertThat(e.getMessage(), containsString("partial")); + } finally { + console.close(); + } + } + + @Test + public void shouldShutdownCleanly() throws Exception { + final String fullStdout = "gremlin>"; + final GremlinConsole console = createConsole(fullStdout, ""); + console.close(); + // If we get here without hanging, shutdown is clean + } + + private GremlinConsole createConsole(final String stdoutContent, final String stderrContent) + throws IOException, GremlinConsole.ConsoleTimeoutException { + return createConsoleWithStdin(stdoutContent, stderrContent, new ByteArrayOutputStream()); + } + + private GremlinConsole createConsoleWithStdin(final String stdoutContent, final String stderrContent, + final OutputStream stdinCapture) + throws IOException, GremlinConsole.ConsoleTimeoutException { + final InputStream stdoutStream = new ByteArrayInputStream(stdoutContent.getBytes(StandardCharsets.UTF_8)); + final InputStream stderrStream = new ByteArrayInputStream(stderrContent.getBytes(StandardCharsets.UTF_8)); + final Process mockProcess = new MockProcess(stdoutStream, stderrStream, stdinCapture); + return new GremlinConsole(mockProcess, 5_000); + } + + /** + * A minimal Process mock that provides controlled streams. + */ + private static class MockProcess extends Process { + private final InputStream stdout; + private final InputStream stderr; + private final OutputStream stdin; + + MockProcess(final InputStream stdout, final InputStream stderr, final OutputStream stdin) { + this.stdout = stdout; + this.stderr = stderr; + this.stdin = stdin; + } + + @Override + public OutputStream getOutputStream() { + return stdin; + } + + @Override + public InputStream getInputStream() { + return stdout; + } + + @Override + public InputStream getErrorStream() { + return stderr; + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public boolean waitFor(final long timeout, final TimeUnit unit) { + return true; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() { + } + + @Override + public Process destroyForcibly() { + return this; + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java new file mode 100644 index 00000000000..76d9188035a --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Options; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Verifies that the AsciidoctorJ extensions register via SPI and process documents correctly. + */ +public class GremlinDocsTest { + + @Test + public void shouldRegisterExtensionsViaSpi() { + // SPI auto-registration happens when Asciidoctor is created + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + assertThat(asciidoctor, is(notNullValue())); + + // Convert a simple doc to verify no errors from extension registration + final String result = asciidoctor.convert("= Test\n\nHello", Options.builder().build()); + assertThat(result, is(notNullValue())); + } + } + + @Test + public void shouldPassthroughPostprocessor() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nSome content here."; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Postprocessor is a no-op, so output should contain the content + assertThat(result.contains("Some content here"), is(true)); + } + } + + @Test + public void shouldFindGremlinGroovyListingBlocks() { + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final String input = "= Test\n\n[gremlin-groovy]\n----\ng.V()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(treeprocessor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldIgnoreNonGremlinListingBlocks() { + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final String input = "= Test\n\n[source,java]\n----\nSystem.out.println();\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(treeprocessor.getGremlinBlockCount(), is(0)); + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java new file mode 100644 index 00000000000..ca2775d0214 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinPostprocessorTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for the GremlinPostprocessor HTML transforms. + */ +public class GremlinPostprocessorTest { + + @Test + public void shouldNotModifyConumElements() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\n" + + "[source,java]\n----\nint x = 1; // <1>\n----\n<1> Sets x\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("conum")); + } + } + + @Test + public void shouldRemoveEmptyCommentSpans() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + // Simulate by testing the postprocessor directly with a mock-like Document + final GremlinPostprocessor pp = new GremlinPostprocessor(); + // Create a minimal document via Asciidoctor for version resolution + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + asciidoctor.load("= T\n\nhi", options); + + // Test the pattern directly + final String html = "code/* */more"; + // We can't easily call process() with a Document without SPI, + // so test the pattern logic + assertThat(html.replaceAll("/\\*\\s*\\*/", ""), + is("codemore")); + } + } + + @Test + public void shouldReplaceVersionPlaceholder() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n:tinkerpop-version: 3.7.7\n\nDownload version x.y.z from the repo."; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("3.7.7")); + assertThat(result, not(containsString("x.y.z"))); + } + } + + @Test + public void shouldNotModifyUnrelatedContent() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nHello world paragraph."; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("tinkerpop-version", "3.7.7").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("Hello world paragraph")); + } + } + + @Test + public void shouldLeaveVersionWhenNoAttribute() { + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + final String input = "= Test\n\nversion x.y.z here"; + // No tinkerpop-version or revnumber set + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("x.y.z")); + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java new file mode 100644 index 00000000000..e22dcb551cd --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeprocessorTest.java @@ -0,0 +1,629 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link GremlinTreeprocessor} that verify block processing logic + * without requiring a real Gremlin Console subprocess. + */ +public class GremlinTreeprocessorTest { + + @Test + public void shouldDetectSugarSyntax() { + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V"), is(true)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V.name"), is(true)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V.outE.weight"), is(true)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V[0..2]"), is(true)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V[0..<2]"), is(true)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.E"), is(true)); + // Normal (non-sugar) statements + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V()"), is(false)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V().values('name')"), is(false)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.V(1).out()"), is(false)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("g.E().count()"), is(false)); + assertThat(GremlinTreeprocessor.usesSugarSyntax("graph = TinkerFactory.createModern()"), is(false)); + } + + @Test + public void shouldExecuteModernGraphBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(1)); + assertThat(executor.statements.contains("graph = TinkerFactory.createModern()"), is(true)); + assertThat(executor.statements.contains("g = graph.traversal()"), is(true)); + assertThat(executor.statements.contains("g.V(1)"), is(true)); + } + } + + @Test + public void shouldUseTinkerGraphForBareBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>2"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy]\n----\n1+1\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("graph = TinkerGraph.open()"), is(true)); + assertThat(executor.statements.contains("g = graph.traversal()"), is(true)); + } + } + + @Test + public void shouldReuseGraphStateForExisting() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,existing]\n----\ng.E()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(2)); + // Should only have one graph init (for modern), not for existing + final long initCount = executor.statements.stream() + .filter(s -> s.startsWith("graph = ")).count(); + assertThat(initCount, is(1L)); + } + } + + @Test + public void shouldReInitGraphForEachBlock() { + // Each non-'existing' block must re-init graph and g, because a prior + // block may have mutated graph or reassigned g (e.g. withComputer()). + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(2)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(2)); + final long modernInitCount = executor.statements.stream() + .filter(s -> s.equals("graph = TinkerFactory.createModern()")).count(); + assertThat(modernInitCount, is(2L)); + } + } + + @Test + public void shouldReInitWhenGraphChanges() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-groovy,classic]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("graph = TinkerFactory.createModern()"), is(true)); + assertThat(executor.statements.contains("graph = TinkerFactory.createClassic()"), is(true)); + } + } + + @Test + public void shouldFormatOutputWithPromptAndTabs() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("gremlin")); + } + } + + @Test + public void shouldFailBuildWhenStatementErrors() { + // A statement that raises a Gremlin error (signalled by the console via a + // GremlinExecutionException) must fail the docs build rather than render as a + // silently-empty block. + final String erroringStatement = "g.V().fail('we failed!')"; + final GremlinTreeprocessor.StatementExecutor erroringExecutor = statement -> { + if (statement.equals(erroringStatement)) { + throw new GremlinConsole.GremlinExecutionException(statement, "fail() Step Triggered"); + } + return "==>ok"; + }; + final GremlinTreeprocessor processor = new GremlinTreeprocessor(erroringExecutor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\n" + erroringStatement + "\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + throw new AssertionError("Expected build to fail with GremlinExecutionException"); + } catch (final GremlinConsole.GremlinExecutionException e) { + assertThat(e.getMessage(), containsString(erroringStatement)); + } + } + + @Test + public void shouldNotFailBuildForEmptyResultBlock() { + // A block that legitimately produces no ==> result (e.g. a void iterate() or an empty + // service list) is NOT an error and must build successfully. + final RecordingExecutor executor = new RecordingExecutor(""); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V().iterate()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(processor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldHandleMultiLineCode() { + final List responses = new ArrayList<>(); + responses.add("==>v[1]\n==>v[2]"); + responses.add("==>2"); + final RecordingExecutor executor = new RecordingExecutor(responses); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V()\ng.V().count()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, is(notNullValue())); + assertThat(executor.statements.contains("g.V()"), is(true)); + assertThat(executor.statements.contains("g.V().count()"), is(true)); + } + } + + @Test + public void shouldKeepMultiLineClosureAsSingleStatement() { + // A Groovy closure whose closing line is not indented (e.g. "}; []") must stay grouped + // with its opening line; splitting it would send an incomplete statement to the console + // and hang at a continuation prompt. + final String[] lines = { + "(1..10).each {", + " g.addV(\"product\").property(\"name\",\"product #${it}\").iterate()", + "}; []", + "g.V().count()" + }; + final List statements = GremlinTreeprocessor.buildStatements(lines); + assertThat(statements.size(), is(2)); + assertThat(statements.get(0), containsString("(1..10).each {")); + assertThat(statements.get(0), containsString("}; []")); + assertThat(statements.get(1), is("g.V().count()")); + } + + @Test + public void shouldNotCountBracketsInsideStrings() { + // Brackets inside string literals (including GString interpolation) must not affect + // statement grouping. + final String[] lines = { + "x = '}'", + "y = \"${a}\"", + "z = 1" + }; + final List statements = GremlinTreeprocessor.buildStatements(lines); + assertThat(statements.size(), is(3)); + } + + @Test + public void shouldHandleCrewGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,crew]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createTheCrew()"), is(true)); + } + } + + @Test + public void shouldHandleGratefulGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,grateful]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createGratefulDead()"), is(true)); + } + } + + @Test + public void shouldHandleSinkGraph() { + final RecordingExecutor executor = new RecordingExecutor("==>result"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,sink]\n----\ng.V()\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(executor.statements.contains("graph = TinkerFactory.createKitchenSink()"), is(true)); + } + } + + @Test + public void shouldFormatDryRunWithPromptsOnly() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\ng.E()\n----\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("gremlin-docs-dryrun", "").build()) + .build(); + final String result = asciidoctor.convert(input, options); + assertThat(result, containsString("gremlin")); + assertThat(result, containsString("gremlin")); + assertThat(processor.getGremlinBlockCount(), is(1)); + } + } + + @Test + public void shouldNotExecuteInDryRunMode() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final Options options = Options.builder() + .attributes(Attributes.builder().attribute("gremlin-docs-dryrun", "").build()) + .build(); + asciidoctor.convert(input, options); + assertThat(executor.statements.isEmpty(), is(true)); + } + } + + @Test + public void shouldCountBlocksWithoutConsole() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy]\n----\ng.V()\n----\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(processor.getGremlinBlockCount(), is(2)); + } + } + + @Test + public void shouldGenerateTabbedHtmlForGremlinBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("section class=\"tabs tabs-2\"")); + assertThat(result, containsString("console (groovy)")); + assertThat(result, containsString("tab-group-1")); + } + } + + @Test + public void shouldConsumeManualTabBlocksAfterGremlinBlock() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n" + + "[source,java]\n----\ng.V(1)\n----\n" + + "[source,python]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("tabs tabs-4")); + assertThat(result, containsString("console (groovy)")); + assertThat(result, containsString("tab-label-3\">java")); + assertThat(result, containsString("tab-label-4\">python")); + } + } + + @Test + public void shouldNotConsumeUnsupportedLanguageAsManualTab() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n" + + "[source,ruby]\n----\ng.V(1)\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Only the console tab, ruby is not consumed + assertThat(result, containsString("tabs tabs-2")); + } + } + + @Test + public void shouldProcessStandaloneTabGroup() { + final GremlinTreeprocessor processor = new GremlinTreeprocessor(); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[source,groovy,tab]\n----\ng.V()\n----\n" + + "[source,java,tab]\n----\ng.V()\n----\n" + + "[source,python,tab]\n----\ng.V()\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + assertThat(result, containsString("tabs tabs-3")); + assertThat(result, containsString("tab-label-1\">groovy")); + assertThat(result, containsString("tab-label-2\">java")); + assertThat(result, containsString("tab-label-3\">python")); + } + } + + @Test + public void shouldKeepStandaloneTabGroupsSeparateFromGremlinGroups() { + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[source,groovy,tab]\n----\ncode1\n----\n" + + "[source,java,tab]\n----\ncode2\n----\n"; + final String result = asciidoctor.convert(input, Options.builder().build()); + // Should have two separate tab groups + assertThat(result, containsString("tab-group-1")); + assertThat(result, containsString("tab-group-2")); + } + } + + @Test + public void shouldInvokeRestartHandlerWhenPluginExcludeAttributePresent() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin,spark-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + assertThat(restartCalls.get(0).contains("neo4j-gremlin"), is(true)); + assertThat(restartCalls.get(0).contains("spark-gremlin"), is(true)); + } + } + + @Test + public void shouldNotInvokeRestartHandlerWhenNoExcludeAttribute() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.isEmpty(), is(true)); + } + } + + @Test + public void shouldNotInvokeRestartHandlerWhenExcludeListUnchanged() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // Process same document twice - handler should only be called once + final String input = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + } + } + + @Test + public void shouldInvokeRestartHandlerWhenExcludeListChanges() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + final String input1 = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input1, Options.builder().build()); + + final String input2 = "= Test\n:gremlin-docs-plugins-exclude: spark-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input2, Options.builder().build()); + assertThat(restartCalls.size(), is(2)); + assertThat(restartCalls.get(0).contains("neo4j-gremlin"), is(true)); + assertThat(restartCalls.get(1).contains("spark-gremlin"), is(true)); + } + } + + @Test + public void shouldInvokeRestartHandlerForSectionLevelExcludeWithinOneDocument() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // Single document with per-section exclusions changing mid-document, as in the reference book. + final String input = "= Book\n\n" + + "== Neo4j\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-docs-plugins-exclude=\"neo4j-gremlin\"]\n" + + "== Spark\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + assertThat(restartCalls.get(0).contains("neo4j-gremlin"), is(true)); + } + } + + @Test + public void shouldLatchSectionExclusionUntilChanged() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // Two consecutive excluding sections sharing the same set => one restart; + // a later section with no attribute inherits (no extra restart). + final String input = "= Book\n\n" + + "[gremlin-docs-plugins-exclude=\"neo4j-gremlin\"]\n" + + "== Hadoop\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "[gremlin-docs-plugins-exclude=\"neo4j-gremlin\"]\n" + + "== Spark\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n\n" + + "== Compilers\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input, Options.builder().build()); + assertThat(restartCalls.size(), is(1)); + } + } + + @Test + public void shouldParseExcludeListWithWhitespace() { + final List result = GremlinTreeprocessor.parseExcludeList(" neo4j-gremlin , spark-gremlin , hadoop-gremlin "); + assertThat(result.size(), is(3)); + assertThat(result.contains("neo4j-gremlin"), is(true)); + assertThat(result.contains("spark-gremlin"), is(true)); + assertThat(result.contains("hadoop-gremlin"), is(true)); + } + + @Test + public void shouldParseEmptyExcludeList() { + assertThat(GremlinTreeprocessor.parseExcludeList("").isEmpty(), is(true)); + assertThat(GremlinTreeprocessor.parseExcludeList(" ").isEmpty(), is(true)); + assertThat(GremlinTreeprocessor.parseExcludeList(null).isEmpty(), is(true)); + } + + @Test + public void shouldInvokeRestartHandlerWithEmptyListWhenAttributeRemoved() { + final List> restartCalls = new ArrayList<>(); + final ConsoleRestartHandler handler = restartCalls::add; + final RecordingExecutor executor = new RecordingExecutor("==>v[1]"); + final GremlinTreeprocessor processor = new GremlinTreeprocessor(executor, handler); + + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.unregisterAllExtensions(); + asciidoctor.javaExtensionRegistry().treeprocessor(processor); + // First doc has exclusions + final String input1 = "= Test\n:gremlin-docs-plugins-exclude: neo4j-gremlin\n\n" + + "[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input1, Options.builder().build()); + + // Second doc has no exclusions + final String input2 = "= Test\n\n[gremlin-groovy,modern]\n----\ng.V(1)\n----\n"; + asciidoctor.convert(input2, Options.builder().build()); + assertThat(restartCalls.size(), is(2)); + assertThat(restartCalls.get(1).isEmpty(), is(true)); + } + } + + /** + * A recording executor that captures all executed statements and returns canned responses. + */ + private static class RecordingExecutor implements GremlinTreeprocessor.StatementExecutor { + final List statements = new ArrayList<>(); + private final List responses; + private int responseIndex = 0; + + RecordingExecutor(final String response) { + this.responses = new ArrayList<>(); + if (response != null) { + this.responses.add(response); + } + } + + RecordingExecutor(final List responses) { + this.responses = responses; + } + + @Override + public String execute(final String statement) { + statements.add(statement); + if (responses.isEmpty()) { + return ""; + } + final String response = responses.get(Math.min(responseIndex, responses.size() - 1)); + responseIndex++; + return response; + } + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java new file mode 100644 index 00000000000..c4318d94113 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/IntegrationTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Attributes; +import org.asciidoctor.Options; +import org.asciidoctor.SafeMode; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Integration test that processes the full fixture document through Asciidoctor with the + * GremlinTreeprocessor (dry-run) and GremlinPostprocessor, verifying the output HTML. + */ +public class IntegrationTest { + + private static String html; + + @BeforeClass + public static void processFixture() throws IOException { + final String fixture; + try (final InputStream is = IntegrationTest.class.getResourceAsStream("/integration-fixture.asciidoc")) { + fixture = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + + final GremlinTreeprocessor treeprocessor = new GremlinTreeprocessor(); + try (final Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + asciidoctor.javaExtensionRegistry().treeprocessor(treeprocessor); + final Options options = Options.builder() + .safe(SafeMode.UNSAFE) + .attributes(Attributes.builder() + .attribute("gremlin-docs-dryrun", "") + .attribute("tinkerpop-version", "3.7.7-SNAPSHOT") + .build()) + .build(); + html = asciidoctor.convert(fixture, options); + } + } + + @Test + public void shouldContainTabbedSectionsForGremlinBlocks() { + assertThat(html, containsString("section class=\"tabs")); + } + + @Test + public void shouldContainGremlinPrompts() { + assertThat(html, containsString("gremlin")); + assertThat(html, containsString("g.V")); + } + + @Test + public void shouldContainConsoleTabLabel() { + assertThat(html, containsString("console (groovy)")); + } + + @Test + public void shouldContainManualTabLabels() { + assertThat(html, containsString(">java<")); + assertThat(html, containsString(">python<")); + } + + @Test + public void shouldProduceMultiTabGroupForManualTabs() { + // The gremlin block + java + python = 3 tabs + assertThat(html, containsString("tabs tabs-3")); + } + + @Test + public void shouldProduceStandaloneTabGroup() { + // groovy + java + csharp standalone tabs = 3 tabs + assertThat(html, containsString(">groovy<")); + assertThat(html, containsString(">csharp<")); + } + + @Test + public void shouldHandleBareGremlinBlock() { + assertThat(html, containsString("integer")); + } + + @Test + public void shouldHandleExistingGraphBlock() { + assertThat(html, containsString("count")); + } + + @Test + public void shouldHandleErrorBlock() { + assertThat(html, containsString("invalid_syntax_here")); + } + + @Test + public void shouldHandleCallouts() { + // Callouts in the [source,java] block should be processed by Asciidoctor + assertThat(html, containsString("conum")); + } + + @Test + public void shouldReplaceVersionPlaceholder() { + assertThat(html, containsString("3.7.7-SNAPSHOT")); + assertThat(html, not(containsString("x.y.z"))); + } + + @Test + public void shouldContainCodeRayHighlightStructure() { + assertThat(html, containsString("CodeRay highlight")); + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandlerTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandlerTest.java new file mode 100644 index 00000000000..d42367cb6a3 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/PluginDirectoryRestartHandlerTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link PluginDirectoryRestartHandler} verifying that toggling plugin + * directories is idempotent and resilient to stale {@code ext-disabled/} state left by an + * interrupted build. + */ +public class PluginDirectoryRestartHandlerTest { + + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + + private static final String NEO4J = "neo4j-gremlin"; + private static final String SPARK = "spark-gremlin"; + private static final String NEO4J_CLASS = "org.apache.tinkerpop.gremlin.neo4j.jsr223.Neo4jGremlinPlugin"; + private static final String SPARK_CLASS = "org.apache.tinkerpop.gremlin.spark.jsr223.SparkGremlinPlugin"; + + private Path consoleHome; + private Path ext; + private Path disabled; + private Path pluginsTxt; + private PluginDirectoryRestartHandler handler; + + @Before + public void setUp() throws IOException { + consoleHome = tmp.getRoot().toPath(); + ext = Files.createDirectories(consoleHome.resolve("ext")); + disabled = consoleHome.resolve("ext-disabled"); + pluginsTxt = ext.resolve("plugins.txt"); + // Seed a populated plugin layout with a non-empty plugin dir for each toggleable plugin. + for (final String p : Arrays.asList(NEO4J, SPARK, "hadoop-gremlin")) { + installPlugin(p); + } + Files.write(pluginsTxt, Arrays.asList( + "org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin", + NEO4J_CLASS, SPARK_CLASS, + "org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopGremlinPlugin")); + handler = new PluginDirectoryRestartHandler(consoleHome); + } + + private void installPlugin(final String plugin) throws IOException { + final Path dir = Files.createDirectories(ext.resolve(plugin).resolve("plugin")); + Files.write(dir.resolve(plugin + ".jar"), "jar-bytes".getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void shouldDisableAndReEnablePlugin() throws IOException { + handler.onRestart(Collections.singletonList(SPARK)); + assertThat(Files.isDirectory(ext.resolve(SPARK)), is(false)); + assertThat(Files.isDirectory(disabled.resolve(SPARK)), is(true)); + assertThat(pluginClasses().contains(SPARK_CLASS), is(false)); + + // A later book with no exclusions restores everything. + handler.onRestart(Collections.emptyList()); + assertThat(Files.isDirectory(ext.resolve(SPARK)), is(true)); + assertThat(Files.isDirectory(disabled.resolve(SPARK)), is(false)); + assertThat(pluginClasses().contains(SPARK_CLASS), is(true)); + // The plugin jar survived the round trip. + assertThat(Files.exists(ext.resolve(SPARK).resolve("plugin").resolve(SPARK + ".jar")), is(true)); + } + + @Test + public void shouldBeIdempotentWhenExcludingTwice() throws IOException { + handler.onRestart(Collections.singletonList(NEO4J)); + // Excluding the same plugin again must not throw, even though ext-disabled/neo4j already exists. + handler.onRestart(Collections.singletonList(NEO4J)); + assertThat(Files.isDirectory(ext.resolve(NEO4J)), is(false)); + assertThat(Files.isDirectory(disabled.resolve(NEO4J)), is(true)); + } + + @Test + public void shouldRecoverFromStaleDisabledDirectoryLeftByInterruptedRun() throws IOException { + // Simulate a crashed prior run: a non-empty ext-disabled/neo4j exists AND ext/neo4j was + // re-installed by process-docs.sh, so the plugin is present in BOTH locations. + Files.createDirectories(disabled.resolve(NEO4J).resolve("plugin")); + Files.write(disabled.resolve(NEO4J).resolve("plugin").resolve("stale.jar"), + "stale".getBytes(StandardCharsets.UTF_8)); + + // Disabling must not throw (the move target already exists and is non-empty). + handler.onRestart(Collections.singletonList(NEO4J)); + assertThat(Files.isDirectory(ext.resolve(NEO4J)), is(false)); + assertThat(Files.isDirectory(disabled.resolve(NEO4J)), is(true)); + // The authoritative active copy replaced the stale one (no stale.jar remains). + assertThat(Files.exists(disabled.resolve(NEO4J).resolve("plugin").resolve("stale.jar")), is(false)); + assertThat(Files.exists(disabled.resolve(NEO4J).resolve("plugin").resolve(NEO4J + ".jar")), is(true)); + } + + @Test + public void shouldEnableCleanlyWhenPluginPresentInBothLocations() throws IOException { + // ext-disabled/spark left over AND ext/spark freshly installed: enabling drops the duplicate. + Files.createDirectories(disabled.resolve(SPARK).resolve("plugin")); + Files.write(disabled.resolve(SPARK).resolve("plugin").resolve("stale.jar"), + "stale".getBytes(StandardCharsets.UTF_8)); + + handler.onRestart(Collections.emptyList()); + assertThat(Files.isDirectory(ext.resolve(SPARK)), is(true)); + assertThat(Files.isDirectory(disabled.resolve(SPARK)), is(false)); + assertThat(pluginClasses().contains(SPARK_CLASS), is(true)); + } + + private List pluginClasses() throws IOException { + return Files.readAllLines(pluginsTxt); + } +} diff --git a/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java new file mode 100644 index 00000000000..20e5f596224 --- /dev/null +++ b/tools/tinkerpop-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/TabbedHtmlBuilderTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Unit tests for {@link TabbedHtmlBuilder}. + */ +public class TabbedHtmlBuilderTest { + + private TabbedHtmlBuilder builder; + + @Before + public void setUp() { + builder = new TabbedHtmlBuilder(); + } + + @Test + public void shouldReturnEmptyStringForNullTabs() { + assertThat(builder.build(null), is("")); + } + + @Test + public void shouldReturnEmptyStringForEmptyTabs() { + assertThat(builder.build(Collections.emptyList()), is("")); + } + + @Test + public void shouldGenerateSectionWithTabsClass() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "gremlin> g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateCorrectTabCountClass() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "g.V()"), + TabbedHtmlBuilder.codeTab("python", "g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateRadioInputsWithCorrectAttributes() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("type=\"radio\"")); + assertThat(html, containsString("name=\"tab-group-1\"")); + assertThat(html, containsString("id=\"tab-1-1\"")); + assertThat(html, containsString("id=\"tab-1-2\"")); + assertThat(html, containsString("class=\"tab-selector-1\"")); + assertThat(html, containsString("class=\"tab-selector-2\"")); + } + + @Test + public void shouldCheckFirstTabByDefault() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("id=\"tab-1-1\" class=\"tab-selector-1\" checked")); + assertThat(html, not(containsString("id=\"tab-1-2\" class=\"tab-selector-2\" checked"))); + } + + @Test + public void shouldGenerateLabelsWithCorrectForAttribute() { + final List tabs = Arrays.asList( + TabbedHtmlBuilder.consoleTab("groovy", "output"), + TabbedHtmlBuilder.codeTab("java", "code")); + final String html = builder.build(tabs); + assertThat(html, containsString("")); + assertThat(html, containsString("")); + } + + @Test + public void shouldNotGenerateClearShadowDiv() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "output")); + final String html = builder.build(tabs); + assertThat(html, not(containsString("
"))); + } + + @Test + public void shouldGenerateTabContentContainer() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.consoleTab("groovy", "output")); + final String html = builder.build(tabs); + assertThat(html, containsString("
")); + assertThat(html, containsString("
")); + } + + @Test + public void shouldGenerateCodeRayPreBlocks() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.codeTab("java", "g.V()")); + final String html = builder.build(tabs); + assertThat(html, containsString("
"));
+        assertThat(html, containsString("
")); + } + + @Test + public void shouldEscapeHtmlInContent() { + final List tabs = Collections.singletonList( + TabbedHtmlBuilder.codeTab("java", "List x = new ArrayList<>()")); + final String html = builder.build(tabs); + assertThat(html, containsString("List<String> x = new ArrayList<>()")); + } + + @Test + public void shouldEscapeHtmlInLabels() { + final TabbedHtmlBuilder.Tab tab = new TabbedHtmlBuilder.Tab("