taskgraph package

taskgraph package#

Subpackages#

Submodules#

taskgraph.config module#

class taskgraph.config.GraphConfig(_config: dict, root_dir: str)#

Bases: object

property docker_dir#

get(name, default=None)#

property kinds_dir#

register()#: Add the project’s taskgraph directory to the python path, and register any extensions present.

root_dir: str#

property taskcluster_yml#

property vcs_root#

class taskgraph.config.GraphConfigSchema(*, trust_domain: str, task_priority: ~typing.Annotated[~typing.Any, <taskgraph.util.schema.OptionallyKeyedBy object at 0x77d558ec30b0>], workers: ~taskgraph.config.WorkersConfig, taskgraph: ~taskgraph.config.TaskgraphConfig, docker_image_kind: str | None = None, task_deadline_after: ~typing.Annotated[~typing.Any, <taskgraph.util.schema.OptionallyKeyedBy object at 0x77d558ec3080>] | None = None, task_expires_after: str | None = None)#

Bases: Schema

docker_image_kind: str | None#

task_deadline_after: OptionallyKeyedBy object at 0x77d558ec3080>] | None#

task_expires_after: str | None#

task_priority: OptionallyKeyedBy object at 0x77d558ec30b0>]#

taskgraph: TaskgraphConfig#

trust_domain: str#

workers: WorkersConfig#

class taskgraph.config.RepositoryConfig(*, name: str, project_regex: str | None = None, ssh_secret_name: str | None = None)#

Bases: Schema

name: str#

project_regex: str | None#

ssh_secret_name: str | None#

class taskgraph.config.RunConfig(use_caches: bool | list[Literal['cargo', 'checkout', 'npm', 'pip', 'uv']] | None = None)#

Bases: Schema

use_caches: bool | list[Literal['cargo', 'checkout', 'npm', 'pip', 'uv']] | None#

class taskgraph.config.TaskgraphConfig(repositories: dict[str, RepositoryConfig], register: str | None = None, decision_parameters: str | None = None, cached_task_prefix: str | None = None, cache_pull_requests: bool | None = None, index_path_regexes: list[str] | None = None, run: RunConfig | None = None, disabled_actions: list[str] | None = None)#

Bases: Schema

cache_pull_requests: bool | None#

cached_task_prefix: str | None#

decision_parameters: str | None#

disabled_actions: list[str] | None#

index_path_regexes: list[str] | None#

register: str | None#

repositories: dict[str, RepositoryConfig]#

run: RunConfig | None#

class taskgraph.config.WorkerAlias(provisioner: ~typing.Annotated[~typing.Any, <taskgraph.util.schema.OptionallyKeyedBy object at 0x77d558ec3710>], implementation: str, os: str, worker_type: ~typing.Annotated[~typing.Any, <taskgraph.util.schema.OptionallyKeyedBy object at 0x77d558ec2a20>])#

Bases: Schema

implementation: str#

os: str#

provisioner: OptionallyKeyedBy object at 0x77d558ec3710>]#

worker_type: OptionallyKeyedBy object at 0x77d558ec2a20>]#

class taskgraph.config.WorkersConfig(aliases: dict[str, WorkerAlias])#

Bases: Schema

aliases: dict[str, WorkerAlias]#

taskgraph.config.graph_config_schema#: alias of GraphConfigSchema

taskgraph.config.load_graph_config(root_dir)#

taskgraph.config.validate_graph_config(config)#

taskgraph.create module#

exception taskgraph.create.CreateTasksException(errors: dict[str, Exception])#

Bases: Exception

Exception raised when one or more tasks could not be created.

taskgraph.create.create_task(session, task_id, label, task_def)#

taskgraph.create.create_tasks(graph_config, taskgraph, label_to_taskid, params, decision_task_id)#

taskgraph.decision module#

class taskgraph.decision.TryTaskConfigSchemaV2(parameters: dict[str, object] | None = None)#

Bases: Schema

parameters: dict[str, object] | None#

taskgraph.decision.full_task_graph_to_runnable_tasks(full_task_json)#

taskgraph.decision.get_decision_parameters(graph_config, options)#: Load parameters from the command-line options for ‘taskgraph decision’. This also applies per-project parameters, based on the given project.

taskgraph.decision.read_artifact(filename)#

taskgraph.decision.rename_artifact(src, dest)#

taskgraph.decision.set_try_config(parameters, task_config_file)#

taskgraph.decision.taskgraph_decision(options, parameters=None)#

Run the decision task. This function implements mach taskgraph decision, and is responsible for

processing decision task command-line options into parameters

running task-graph generation exactly the same way the other mach taskgraph commands do

generating a set of artifacts to memorialize the graph

calling TaskCluster APIs to create the graph

taskgraph.decision.try_task_config_schema_v2#: alias of TryTaskConfigSchemaV2

taskgraph.decision.write_artifact(filename, data)#

taskgraph.docker module#

taskgraph.docker.build_image(graph_config: GraphConfig, name: str, context_file: str | None = None, save_image: str | None = None) → str#

Build a Docker image of specified name.

Builds a Docker image from the specified image directory.

Parameters:

graph_config – The graph configuration.
name – The name of the Docker image to build.
context_file – Path to save the docker context to. If specified, only the context is generated and the image isn’t built.
save_image – If specified, the resulting image.tar will be saved to the specified path. Otherwise, the image is loaded into docker.

Returns:

The tag of the loaded image, or absolute path to the image: if save_image is specified.

Return type:

str

Raises:

ValueError – If name is not provided.
Exception – If the image directory does not exist.

taskgraph.docker.get_image_digest(image_name: str) → str#

Get the digest of a docker image by its name.

Parameters:: image_name – The name of the docker image to get the digest for.
Returns:: The digest string of the cached docker image task.
Return type:: str

taskgraph.docker.load_image(url: str, imageName: str | None = None, imageTag: str | None = None) → dict[str, str]#

Load docker image from URL as imageName:tag.

Downloads a zstd-compressed docker image tarball from the given URL and loads it into the local Docker daemon. If no imageName or tag is given, it will use whatever is inside the compressed tarball.

Parameters:

url – URL to download the zstd-compressed docker image from.
imageName – Optional name to give the loaded image. If provided without imageTag, will parse tag from name or default to ‘latest’.
imageTag – Optional tag to give the loaded image.

Returns:

An object with properties ‘image’, ‘tag’ and ‘layer’ containing: information about the loaded image.

Return type:

dict

Raises:

ImportError – If zstandard package is not installed.
Exception – If the tar contains multiple images/tags or no repositories file.

taskgraph.docker.load_image_by_name(image_name: str, tag: str | None = None) → str | None#

Load a docker image by its name.

Finds the appropriate docker image task by name and loads it from the indexed artifacts.

Parameters:

image_name – The name of the docker image to load.
tag – Optional tag to apply to the loaded image. If not provided, uses the tag from the image artifact.

Returns:

The full image tag (name:tag) if successful, None if: the image artifacts could not be found.

Return type:

str or None

taskgraph.docker.load_image_by_task_id(task_id: str, tag: str | None = None) → str#

Load a docker image from a task’s artifacts.

Downloads and loads a docker image from the specified task’s public/image.tar.zst artifact.

Parameters:

task_id – The task ID containing the docker image artifact.
tag – Optional tag to apply to the loaded image. If not provided, uses the tag from the image artifact.

Returns:

The full image tag (name:tag) that was loaded.

Return type:

str

taskgraph.docker.load_task(graph_config: GraphConfig, task: str | dict[str, Any], remove: bool = True, user: str | None = None, custom_image: str | None = None, interactive: bool | None = False, volumes: list[tuple[str, str]] | None = None, develop: bool = False) → int#

Load and run a task interactively in a Docker container.

Downloads the docker image from a task’s definition and runs it in an interactive shell, setting up the task environment but not executing the actual task command. The task command can be executed later using the ‘exec-task’ function provided in the shell.

Parameters:

graph_config – The graph configuration object.
task – The ID of the task, or task definition to load.
remove – Whether to remove the container after exit (default True).
user – The user to switch to in the container (default ‘worker’).
custom_image – A custom image to use instead of the task’s image.
interactive – If True, execution of the task will be paused and user will be dropped into a shell. They can run exec-task to resume it (default: False).
develop – If True, the task will be configured to use the current local checkout at the current revision (default: False).

Returns:

The exit code from the Docker container.

Return type:

int

taskgraph.filter_tasks module#

taskgraph.filter_tasks.filter_target_tasks(graph, parameters, graph_config)#

Proxy filter to use legacy target tasks code.

This should go away once target_tasks are converted to filters.

taskgraph.filter_tasks.filter_task(name)#: Generator to declare a task filter function.

taskgraph.generator module#

class taskgraph.generator.Kind(name: str, path: str, config: dict, graph_config: taskgraph.config.GraphConfig)#

Bases: object

config: dict#

graph_config: GraphConfig#

classmethod load(root_dir, graph_config, kind_name)#

load_tasks(parameters, kind_dependencies_tasks, write_artifacts)#

name: str#

path: str#

exception taskgraph.generator.KindNotFound#

Bases: Exception

Raised when trying to load kind from a directory without a kind.yml.

class taskgraph.generator.TaskGraphGenerator(root_dir: str | None, parameters: Parameters | Callable[[GraphConfig], Parameters], decision_task_id: str = 'DECISION-TASK', write_artifacts: bool = False, enable_verifications: bool = True)#

Bases: object

The central controller for taskgraph. This handles all phases of graph generation. The task is generated from all of the kinds defined in subdirectories of the generator’s root directory.

Access to the results of this generation, as well as intermediate values at various phases of generation, is available via properties. This encourages the provision of all generation inputs at instance construction time.

property full_task_graph#

the full task set, with edges representing dependencies.

@type: TaskGraph

Type:: The full task graph

property full_task_set#

all tasks defined by any kind (a graph without edges)

@type: TaskGraph

Type:: The full task set

property graph_config#

The configuration for this graph.

@type: TaskGraph

property kind_graph#

The dependency graph of kinds.

@type: Graph

property label_to_taskid#

A dictionary mapping task label to assigned taskId. This property helps in interpreting optimized_task_graph.

@type: dictionary

property morphed_task_graph#

The optimized task graph, with any subsequent morphs applied. This graph will have the same meaning as the optimized task graph, but be in a form more palatable to TaskCluster.

@type: TaskGraph

property optimized_task_graph#

The set of targeted tasks and all of their dependencies; tasks that have been optimized out are either omitted or replaced with a Task instance containing only a task_id.

@type: TaskGraph

property parameters#

The properties used for this graph.

@type: Properties

property target_task_graph#

The set of targeted tasks and all of their dependencies

@type: TaskGraph

property target_task_set#

The set of targeted tasks (a graph without edges)

@type: TaskGraph

verify(name, *args, **kwargs)#

taskgraph.generator.load_tasks_for_kind(parameters, kind, root_dir=None, **tgg_kwargs)#

Get all the tasks of a given kind.

This function is designed to be called from outside of taskgraph.

taskgraph.generator.load_tasks_for_kinds(parameters, kinds, root_dir=None, graph_attr=None, **tgg_kwargs)#

Get all the tasks of the given kinds.

This function is designed to be called from outside of taskgraph.

taskgraph.graph module#

class taskgraph.graph.Graph(nodes, edges)#

Bases: _Graph

Generic representation of a directed acyclic graph with labeled edges connecting the nodes. Graph operations are implemented in a functional manner, so the data structure is immutable.

It permits at most one edge of a given name between any set of nodes. The graph is not checked for cycles, and methods may hang or otherwise fail if given a cyclic graph.

The nodes and edges attributes may be accessed in a read-only fashion. The nodes attribute is a set of node names, while edges is a set of (left, right, name) tuples representing an edge named name going from node left to node right..

links_and_reverse_links_dict()#

Return both links and reverse_links dictionaries. Returns a (forward_links, reverse_links) tuple where forward_links maps each node to the set of nodes it links to, and reverse_links maps each node to the set of nodes linking to it.

Because the return value is cached, this function returns a pair of ReadOnlyDict instead of defaultdicts like its links_dict and reverse_links_dict counterparts to avoid consumers modifying the cached value by mistake.

links_dict()#: Return a dictionary mapping each node to a set of the nodes it links to (omitting edge names)

named_links_dict()#: Return a two-level dictionary mapping each node to a dictionary mapping edge names to labels.

reverse_links_dict()#: Return a dictionary mapping each node to a set of the nodes linking to it (omitting edge names)

transitive_closure(nodes, reverse=False)#

Return the transitive closure of <nodes>: the graph containing all specified nodes as well as any nodes reachable from them, and any intervening edges.

If reverse is true, the “reachability” will be reversed and this will return the set of nodes that can reach the specified nodes.

Example:

a ------> b ------> c
          |
          `-------> d

transitive_closure([b]).nodes == set([a, b]) transitive_closure([c]).nodes == set([c, b, a]) transitive_closure([c], reverse=True).nodes == set([c]) transitive_closure([b], reverse=True).nodes == set([b, c, d])

visit_postorder()#

Generate a sequence of nodes in postorder, such that every node is visited after any nodes it links to.

Raises an exception if the graph contains a cycle.

visit_preorder()#: Like visit_postorder, but in reverse: evrey node is visited before any nodes it links to.

taskgraph.main module#

class taskgraph.main.Command(func, args, kwargs, defaults)#

Bases: tuple

args#: Alias for field number 1

defaults#: Alias for field number 3

func#: Alias for field number 0

kwargs#: Alias for field number 2

taskgraph.main.action_callback(options)#

taskgraph.main.actions(args)#

taskgraph.main.argument(*args, **kwargs)#

taskgraph.main.build_image(args)#

taskgraph.main.command(*args, **kwargs)#

taskgraph.main.create_parser()#

taskgraph.main.decision(options)#

taskgraph.main.dump_output(out, path=None, params_spec=None)#

taskgraph.main.format_kind_graph_mermaid(kind_graph)#

Convert a kind dependency graph to Mermaid flowchart format.

@param kind_graph: Graph object containing kind nodes and dependencies @return: String representation of the graph in Mermaid format

taskgraph.main.format_taskgraph(options, parameters, overrides, logfile=None)#

taskgraph.main.format_taskgraph_json(taskgraph)#

taskgraph.main.format_taskgraph_labels(taskgraph)#

taskgraph.main.format_taskgraph_yaml(taskgraph)#

taskgraph.main.generate_taskgraph(options, parameters, overrides, logdir)#

taskgraph.main.get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys)#: Filter all the tasks on basis of a regular expression and returns a new TaskGraph object

taskgraph.main.get_taskgraph_generator(root, parameters)#: Helper function to make testing a little easier.

taskgraph.main.image_digest(args)#

taskgraph.main.init_taskgraph(options)#

taskgraph.main.load_image(args)#

taskgraph.main.load_task(args)#

taskgraph.main.main(args=['-T', '-b', 'html', '-d', 'docs/_build/doctrees', '-D', 'language=en', 'docs', '/home/docs/checkouts/readthedocs.org/user_builds/taskcluster-taskgraph/checkouts/23.2.0/_readthedocs//html'])#

taskgraph.main.setup_logging()#

taskgraph.main.show_kind_graph(options)#

taskgraph.main.show_taskgraph(options)#

taskgraph.main.test_action_callback(options)#

taskgraph.main.validate_docker()#

taskgraph.morph module#

Graph morphs are modifications to task-graphs that take place after the optimization phase.

These graph morphs are largely invisible to developers running ./mach locally, so they should be limited to changes that do not modify the meaning of the graph.

taskgraph.morph.add_code_review_task(taskgraph, label_to_taskid, parameters, graph_config)#

taskgraph.morph.add_index_tasks(taskgraph, label_to_taskid, parameters, graph_config)#: The TaskCluster queue only allows 64 routes on a task. In the event a task exceeds this limit, this graph morph adds “index tasks” that depend on it and do the index insertions directly, avoiding the limit on task.routes.

taskgraph.morph.amend_taskgraph(taskgraph, label_to_taskid, to_add)#: Add the given tasks to the taskgraph, returning a new taskgraph

taskgraph.morph.derive_index_task(task, taskgraph, label_to_taskid, parameters, graph_config)#: Create the shell of a task that depends on task and on the given docker image.

taskgraph.morph.make_index_task(parent_task, taskgraph, label_to_taskid, parameters, graph_config)#

taskgraph.morph.morph(taskgraph, label_to_taskid, parameters, graph_config)#: Apply all morphs

taskgraph.morph.register_morph(func)#

taskgraph.parameters module#

exception taskgraph.parameters.ParameterMismatch#

Bases: Exception

Raised when a parameters.yml has extra or missing parameters.

class taskgraph.parameters.Parameters(strict=True, repo_root=None, **kwargs)#

Bases: ReadOnlyDict

An immutable dictionary with nicer KeyError messages on failure

check()#

file_url(path, pretty=False)#

Determine the VCS URL for viewing a file in the tree, suitable for viewing by a human.

Parameters:

path (str) – The path, relative to the root of the repository.
pretty (bool) – Whether to return a link to a formatted version of the file, or the raw file version.

Return str:

The URL displaying the given path.

static format_spec(spec)#

Get a friendly identifier from a parameters specifier.

Parameters:: spec (str) – Parameters specifier.
Returns:: Name to identify parameters by.
Return type:: str

property id#

is_try()#: Determine whether this graph is being built on a try project or for mach try fuzzy.

property moz_build_date#

taskgraph.parameters.base_schema#: Schema for base parameters. Please keep this list sorted and in sync with docs/reference/parameters.rst

taskgraph.parameters.extend_parameters_schema(schema, defaults_fn=None)#

Extend the schema for parameters to include per-project configuration.

This should be called by the taskgraph.register function in the graph-configuration.

Parameters:

schema – A msgspec Schema subclass describing extended parameters.
defaults_fn (function) – A function which takes no arguments and returns a dict mapping parameter name to default value in the event strict=False (optional).

taskgraph.parameters.get_contents(path)#

taskgraph.parameters.get_version(repo_path)#

taskgraph.parameters.load_parameters_file(spec, strict=True, overrides=None, trust_domain=None, repo_root=None)#

Load parameters from a path, url, decision task-id or project.

Examples

task-id=fdtgsD5DQUmAQZEaGMvQ4Q project=mozilla-central

taskgraph.parameters.parameters_loader(spec, strict=True, overrides=None)#

taskgraph.target_tasks module#

taskgraph.target_tasks.filter_for_git_branch(task, parameters)#: Filter tasks by git branch. If run_on_git_branch is not defined, then task runs on all branches

taskgraph.target_tasks.filter_for_project(task, parameters)#: Filter tasks by project. Optionally enable nightlies.

taskgraph.target_tasks.filter_for_tasks_for(task, parameters)#

taskgraph.target_tasks.filter_out_cron(task, parameters)#: Filter out tasks that run via cron.

taskgraph.target_tasks.filter_out_shipping_phase(task, parameters)#

taskgraph.target_tasks.get_method(method)#: Get a target_task_method to pass to a TaskGraphGenerator.

taskgraph.target_tasks.register_target_task(name)#

taskgraph.target_tasks.standard_filter(task, parameters)#

taskgraph.target_tasks.target_tasks_codereview(full_task_graph, parameters, graph_config)#: Target the tasks which have indicated they should be run on this project via the run_on_projects attributes.

taskgraph.target_tasks.target_tasks_default(full_task_graph, parameters, graph_config)#: Target the tasks which have indicated they should be run on this project via the run_on_projects attributes.

taskgraph.target_tasks.target_tasks_nothing(full_task_graph, parameters, graph_config)#: Select nothing, for DONTBUILD pushes

taskgraph.task module#

class taskgraph.task.Task(kind: str, label: str, attributes: dict, task: dict, description: str = '', optimization: dict[str, ~typing.Any] | None=None, dependencies: dict = <factory>, soft_dependencies: list = <factory>, if_dependencies: list = <factory>)#

Bases: object

Representation of a task in a TaskGraph. Each Task has, at creation:

kind: the name of the task kind
label; the label for this task
attributes: a dictionary of attributes for this task (used for filtering)
task: the task definition (JSON-able dictionary)
optimization: optimization to apply to the task (see taskgraph.optimize)
dependencies: tasks this one depends on, in the form {name: label}, for example {‘build’: ‘build-linux64/opt’, ‘docker-image’: ‘docker-image-desktop-test’}
soft_dependencies: tasks this one may depend on if they are available post optimisation. They are set as a list of tasks label.
if_dependencies: only run this task if at least one of these dependencies are present.

And later, as the task-graph processing proceeds:

task_id – TaskCluster taskId under which this task will be created

This class is just a convenience wrapper for the data type and managing display, comparison, serialization, etc. It has no functionality of its own.

attributes: dict#

dependencies: dict#

description: str = ''#

classmethod from_json(task_dict)#: Given a data structure as produced by taskgraph.to_json, re-construct the original Task object. This is used to “resume” the task-graph generation process, for example in Action tasks.

if_dependencies: list#

kind: str#

label: str#

optimization: dict[str, Any] | None = None#

soft_dependencies: list#

task: dict#

task_id: str | None = None#

to_json()#

taskgraph.taskgraph module#

class taskgraph.taskgraph.TaskGraph(tasks: dict[str, Task], graph: Graph)#

Bases: object

Representation of a task graph.

A task graph is a combination of a Graph and a dictionary of tasks indexed by label. TaskGraph instances should be treated as immutable.

In the graph, tasks are said to “link to” their dependencies. Whereas tasks are “linked from” their dependents.

for_each_task(f, *args, **kwargs)#

classmethod from_json(tasks_dict)#: This code is used to generate the a TaskGraph using a dictionary which is representative of the TaskGraph.

graph: Graph#

tasks: dict[str, Task]#

to_json()#: Return a JSON-able object representing the task graph, as documented

taskgraph package

Contents

taskgraph package#

Subpackages#

Submodules#

taskgraph.config module#

taskgraph.create module#

taskgraph.decision module#

taskgraph.docker module#

taskgraph.filter_tasks module#

taskgraph.generator module#

taskgraph.graph module#

taskgraph.main module#

taskgraph.morph module#

taskgraph.parameters module#

taskgraph.target_tasks module#

taskgraph.task module#

taskgraph.taskgraph module#

Module contents#