From db29411ca87c163cd751f2a76969dcc2601c6d45 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 24 Jun 2026 12:20:22 -0400 Subject: [PATCH] feat(cli): replace various format flags with `--format={format}` The old flags are left as aliases for backwards compatibility. --- docs/howto/load-task-locally.rst | 6 +- docs/howto/run-locally.rst | 21 ++- docs/index.rst | 2 +- docs/tutorials/creating-a-task-graph.rst | 4 +- src/taskgraph/main.py | 230 ++++++++++++++--------- test/test_main.py | 2 +- 6 files changed, 163 insertions(+), 102 deletions(-) diff --git a/docs/howto/load-task-locally.rst b/docs/howto/load-task-locally.rst index 066360779..2ae9c277c 100644 --- a/docs/howto/load-task-locally.rst +++ b/docs/howto/load-task-locally.rst @@ -55,7 +55,7 @@ For example, you can run: .. code-block:: shell - taskgraph morphed -J --tasks test-unit-py | jq -r 'to_entries | first | .value.task' | taskgraph load-task - + taskgraph morphed --format json --tasks test-unit-py | jq -r 'to_entries | first | .value.task' | taskgraph load-task - In this example, we're piping the definition of the task named ``test-unit-py`` into ``taskgraph load-task``. This allows you to quickly iterate on the task @@ -63,7 +63,7 @@ and test that it runs as expected. .. note:: - The output of ``taskgraph morphed -J`` looks something like: + The output of ``taskgraph morphed --format json`` looks something like: ``{ "": { "task": { }}}`` @@ -103,7 +103,7 @@ by combining passing in a custom image locally, and piping a task definition via .. code-block:: shell - taskgraph morphed -J --tasks test-unit-py | jq -r 'to_entries | first | .value.task' | taskgraph load-task --image python - + taskgraph morphed --format json --tasks test-unit-py | jq -r 'to_entries | first | .value.task' | taskgraph load-task --image python - Developing in the Container --------------------------- diff --git a/docs/howto/run-locally.rst b/docs/howto/run-locally.rst index 84fd5dc60..9c0e8c350 100644 --- a/docs/howto/run-locally.rst +++ b/docs/howto/run-locally.rst @@ -42,17 +42,18 @@ Useful Arguments Here are some useful arguments accepted by most ``taskgraph`` subcommands. For a full reference, see :doc:`/reference/cli`. -``-J/--json`` -+++++++++++++ +``--format`` +++++++++++++ -By default only the task labels are displayed as output, but when ``-J/--json`` -is used, the full JSON representation of all task definitions are displayed. +By default only the task labels are displayed as output, but when +``--format json`` is used, the full JSON representation of all task definitions +are displayed. .. note:: - Using ``-J/--json`` can often result in a massive amount of output. Consider - using the ``--tasks`` and/or ``--target-kind`` flags in conjunction to - filter the result down to a manageable level. + Using ``--format json`` can often result in a massive amount of output. + Consider using the ``--tasks`` and/or ``--target-kind`` flags in conjunction + to filter the result down to a manageable level. ``--tasks/--tasks-regex`` +++++++++++++++++++++++++ @@ -126,7 +127,7 @@ Validating Changes to Task Definitions If you're only modifying the definition of tasks, then you want to generate the ``full_task_graph``. This is because task definitions are frozen (with minor -exceptions) after this phase. You'll also want to use the ``-J/--json`` flag and +exceptions) after this phase. You'll also want to use ``--format json`` and likely also the ``--tasks`` flag to filter down the result. For example, let's say you modify a task called ``build-android``. Then you @@ -134,7 +135,7 @@ would run the following command: .. code-block:: shell - taskgraph full -J --tasks "build-android" + taskgraph full --format json --tasks "build-android" Then you can inspect the resulting task definition and validate that everything is configured as you expect. @@ -148,7 +149,7 @@ If you're modifying *where* a task runs, e.g by changing a key that impacts the ``target_task_graph`` phase. Unlike when modifying the definition, we don't care about the contents of the -task so passing the ``-J/--json`` flag is unnecessary. Instead, we can simply +task so passing ``--format json`` is unnecessary. Instead, we can simply inspect whether the label exists or not. However it *is* important to make sure we're generating under the appropriate context(s) via the ``-p/--parameters`` flag. diff --git a/docs/index.rst b/docs/index.rst index fb4fea4fa..031983dd8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -75,7 +75,7 @@ already, namely ``docker-image-linux`` and ``hello-world``. .. note:: By default the ``taskgraph`` command will only output task labels. Try - adding ``--json`` to the command to see the actual definitions. + adding ``--format json`` to the command to see the actual definitions. See if you can create a new task by editing ``taskcluster/kinds/hello/kind.yml``, and re-run ``taskgraph full`` to verify. diff --git a/docs/tutorials/creating-a-task-graph.rst b/docs/tutorials/creating-a-task-graph.rst index 35be52bc6..19c02300f 100644 --- a/docs/tutorials/creating-a-task-graph.rst +++ b/docs/tutorials/creating-a-task-graph.rst @@ -202,9 +202,9 @@ Now run: .. code-block:: bash - taskgraph morphed -J + taskgraph morphed --format json -The ``-J/--json`` flag will display the full JSON definition of your task. +The ``--format json`` flag will display the full JSON definition of your task. Morphed is the final phase of :ref:`graph generation `, so represents your task's final form before it would get submitted to Taskcluster. In fact, if we hadn't made up the trust domain and worker pool in diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index e74da8a7d..aca68f054 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -142,6 +142,19 @@ def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys): } +def parse_format_spec(value): + """Parse a format specifier like 'json' or 'json:path/to/file.json'.""" + parts = value.split(":", 1) + fmt = parts[0] + if fmt not in FORMAT_METHODS: + valid = ", ".join(sorted(FORMAT_METHODS)) + raise argparse.ArgumentTypeError( + f"unknown format '{fmt}'; must be one of: {valid}" + ) + path = parts[1] if len(parts) > 1 else None + return fmt, path + + def get_taskgraph_generator(root, parameters): """Helper function to make testing a little easier.""" from taskgraph.generator import TaskGraphGenerator # noqa: PLC0415 @@ -175,8 +188,12 @@ def format_taskgraph(options, parameters, overrides, logfile=None): tg = getattr(tgg, options["graph_attr"]) tg = get_filtered_taskgraph(tg, options["tasks_regex"], options["exclude_keys"]) - format_method = FORMAT_METHODS[options["format"] or "labels"] - return format_method(tg) + formats = options.get("formats") or [("labels", None)] + results = {} + for fmt, _ in formats: + if fmt not in results: + results[fmt] = FORMAT_METHODS[fmt](tg) + return results def dump_output(out, path=None, params_spec=None): @@ -201,9 +218,16 @@ def dump_output(out, path=None, params_spec=None): print(out + "\n", file=fh) +def dump_outputs(results, formats, params_spec=None): + for fmt, path in formats: + dump_output(results[fmt], path, params_spec) + + def generate_taskgraph(options, parameters, overrides, logdir): from taskgraph.parameters import Parameters # noqa: PLC0415 + formats = options.get("formats") or [("labels", None)] + def logfile(spec): """Determine logfile given a parameters specification.""" if logdir is None: @@ -217,8 +241,8 @@ def logfile(spec): # tracebacks a little more readable and avoids additional process overhead. if len(parameters) == 1: spec = parameters[0] - out = format_taskgraph(options, spec, overrides, logfile(spec)) - dump_output(out, options["output_file"]) + results = format_taskgraph(options, spec, overrides, logfile(spec)) + dump_outputs(results, formats) return 0 futures = {} @@ -231,23 +255,23 @@ def logfile(spec): futures[f] = spec for future in as_completed(futures): - output_file = options["output_file"] spec = futures.pop(future) + params_spec = spec if len(parameters) > 1 else None e = future.exception() if e: returncode = 1 - out = "".join(traceback.format_exception(type(e), e, e.__traceback__)) + error_out = "".join( + traceback.format_exception(type(e), e, e.__traceback__) + ) if options["diff"]: # Dump to console so we don't accidentally diff the tracebacks. - output_file = None + dump_output(error_out, path=None, params_spec=params_spec) + else: + _, error_path = formats[0] + dump_output(error_out, path=error_path, params_spec=params_spec) else: - out = future.result() - - dump_output( - out, - path=output_file, - params_spec=spec if len(parameters) > 1 else None, - ) + results = future.result() + dump_outputs(results, formats, params_spec=params_spec) return returncode @@ -350,29 +374,42 @@ def show_kind_graph(options): @argument( "--verbose", "-v", action="store_true", help="include debug-level logging output" ) +@argument( + "--format", + "--fmt", + action="append", + dest="formats", + type=parse_format_spec, + default=None, + metavar="FORMAT[:PATH]", + help="Output format and optional destination. FORMAT is one of: labels, json, yaml. " + "If PATH is omitted, output goes to stdout. Can be specified multiple times to " + "produce multiple outputs (e.g. --format json:out.json --format labels). " + "Default: labels (to stdout).", +) @argument( "--json", "-J", action="store_const", - dest="format", + dest="format_compat", const="json", - help="Output task graph as a JSON object", + help=argparse.SUPPRESS, ) @argument( "--yaml", "-Y", action="store_const", - dest="format", + dest="format_compat", const="yaml", - help="Output task graph as a YAML object", + help=argparse.SUPPRESS, ) @argument( "--labels", "-L", action="store_const", - dest="format", + dest="format_compat", const="labels", - help="Output the label for each task in the task graph (default)", + help=argparse.SUPPRESS, ) @argument( "--parameters", @@ -408,8 +445,9 @@ def show_kind_graph(options): @argument( "-o", "--output-file", + dest="output_file_compat", default=None, - help="file path to store generated output.", + help=argparse.SUPPRESS, ) @argument( "--tasks-regex", @@ -466,10 +504,19 @@ def show_taskgraph(options): if options.pop("verbose", False): logging.root.setLevel(logging.DEBUG) + format_compat = options.pop("format_compat", None) + output_file_compat = options.pop("output_file_compat", None) + if format_compat is not None or output_file_compat is not None: + logging.warning( + "--json/--yaml/--labels/--output-file are deprecated; use --format instead" + ) + if not options.get("formats"): + options["formats"] = [(format_compat or "labels", output_file_compat)] + repo = None cur_rev = None diffdir = None - output_file = options["output_file"] + formats = options.get("formats") or [("labels", None)] if options["diff"] or options["force_local_files_changed"]: repo = get_repository(os.getcwd()) @@ -494,9 +541,13 @@ def show_taskgraph(options): atexit.register( shutil.rmtree, diffdir ) # make sure the directory gets cleaned up - options["output_file"] = os.path.join( - diffdir, f"{options['graph_attr']}_{cur_rev_file}" - ) + options["formats"] = [ + ( + fmt, + os.path.join(diffdir, f"{options['graph_attr']}_{cur_rev_file}_{fmt}"), + ) + for fmt, _ in formats + ] print(f"Generating {options['graph_attr']} @ {cur_rev}", file=sys.stderr) overrides = { @@ -560,9 +611,15 @@ def show_taskgraph(options): try: repo.update(base_rev) base_rev = repo.head_rev[:12] - options["output_file"] = os.path.join( - diffdir, f"{options['graph_attr']}_{base_rev_file}" - ) + options["formats"] = [ + ( + fmt, + os.path.join( + diffdir, f"{options['graph_attr']}_{base_rev_file}_{fmt}" + ), + ) + for fmt, _ in formats + ] print(f"Generating {options['graph_attr']} @ {base_rev}", file=sys.stderr) ret |= generate_taskgraph(options, parameters, overrides, logdir) finally: @@ -570,65 +627,68 @@ def show_taskgraph(options): repo.update(cur_rev) # Generate diff(s) - diffcmd = [ - "diff", - "-U20", - "--report-identical-files", - f"--label={options['graph_attr']}@{base_rev}", - f"--label={options['graph_attr']}@{cur_rev}", - ] - non_fatal_failures = [] for spec in parameters: - base_path = os.path.join( - diffdir, f"{options['graph_attr']}_{base_rev_file}" - ) - cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_rev_file}") # type: ignore - - params_name = None - if len(parameters) > 1: - params_name = Parameters.format_spec(spec) - base_path += f"_{params_name}" - cur_path += f"_{params_name}" - - # If the base or cur files are missing it means that generation - # failed. If one of them failed but not the other, the failure is - # likely due to the patch making changes to taskgraph in modules - # that don't get reloaded (safe to ignore). If both generations - # failed, there's likely a real issue. - base_missing = not os.path.isfile(base_path) - cur_missing = not os.path.isfile(cur_path) - if base_missing != cur_missing: # != is equivalent to XOR for booleans - non_fatal_failures.append(os.path.basename(base_path)) - continue - - try: - # If the output file(s) are missing, this command will raise - # CalledProcessError with a returncode > 1. - proc = subprocess.run( - diffcmd + [base_path, cur_path], - capture_output=True, - text=True, - check=True, + params_name = Parameters.format_spec(spec) if len(parameters) > 1 else None + + for fmt, output_path in formats: + base_path = os.path.join( + diffdir, f"{options['graph_attr']}_{base_rev_file}_{fmt}" + ) + cur_path = os.path.join( # type: ignore + diffdir, f"{options['graph_attr']}_{cur_rev_file}_{fmt}" + ) + + if params_name: + base_path += f"_{params_name}" + cur_path += f"_{params_name}" + + # If the base or cur files are missing it means that generation + # failed. If one of them failed but not the other, the failure is + # likely due to the patch making changes to taskgraph in modules + # that don't get reloaded (safe to ignore). If both generations + # failed, there's likely a real issue. + base_missing = not os.path.isfile(base_path) + cur_missing = not os.path.isfile(cur_path) + if base_missing != cur_missing: # != is equivalent to XOR for booleans + non_fatal_failures.append(os.path.basename(base_path)) + continue + + diffcmd = [ + "diff", + "-U20", + "--report-identical-files", + f"--label={options['graph_attr']}:{fmt}@{base_rev}", + f"--label={options['graph_attr']}:{fmt}@{cur_rev}", + ] + + try: + # If the output file(s) are missing, this command will raise + # CalledProcessError with a returncode > 1. + proc = subprocess.run( + diffcmd + [base_path, cur_path], + capture_output=True, + text=True, + check=True, + ) + diff_output = proc.stdout + diff_returncode = 0 + except subprocess.CalledProcessError as e: + # returncode 1 simply means diffs were found + if e.returncode != 1: + print(e.stderr, file=sys.stderr) + raise + diff_output = e.output + diff_returncode = e.returncode + + dump_output( + diff_output, + # Don't bother saving file if no diffs were found. Log to + # console in this case instead. + path=None if diff_returncode == 0 else output_path, + params_spec=spec if len(parameters) > 1 else None, ) - diff_output = proc.stdout - returncode = 0 - except subprocess.CalledProcessError as e: - # returncode 1 simply means diffs were found - if e.returncode != 1: - print(e.stderr, file=sys.stderr) - raise - diff_output = e.output - returncode = e.returncode - - dump_output( - diff_output, - # Don't bother saving file if no diffs were found. Log to - # console in this case instead. - path=None if returncode == 0 else output_file, - params_spec=spec if len(parameters) > 1 else None, - ) if non_fatal_failures: failstr = "\n ".join(sorted(non_fatal_failures)) @@ -640,10 +700,10 @@ def show_taskgraph(options): file=sys.stderr, ) - if options["format"] != "json": + if not any(fmt == "json" for fmt, _ in formats): print( "If you were expecting differences in task bodies " - 'you should pass "-J"\n', + 'you should pass "--format json"\n', file=sys.stderr, ) diff --git a/test/test_main.py b/test/test_main.py index 13c750d95..2a38f2035 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -162,7 +162,7 @@ def test_output_file(run_taskgraph, tmpdir): output_file = tmpdir.join("out.txt") assert not output_file.check() - run_taskgraph(["full", f"--output-file={output_file.strpath}"]) + run_taskgraph(["full", f"--format=labels:{output_file.strpath}"]) assert output_file.check() assert output_file.read_text("utf-8").strip() == "\n".join( ["_fake-t-0", "_fake-t-1", "_fake-t-2"]