I'm trying to write a custom rule to compile C++ code using the cc_common API. Here's my current attempt at an implementation:
load("#bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
load("#bazel_tools//tools/build_defs/cc:action_names.bzl", "C_COMPILE_ACTION_NAME")
def _impl(ctx):
cc_toolchain = find_cpp_toolchain(ctx)
feature_configuration = cc_common.configure_features(
cc_toolchain = cc_toolchain,
unsupported_features = ctx.disabled_features,
)
compiler = cc_common.get_tool_for_action(
feature_configuration=feature_configuration,
action_name=C_COMPILE_ACTION_NAME
)
compile_variables = cc_common.create_compile_variables(
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
)
compiler_options = cc_common.get_memory_inefficient_command_line(
feature_configuration = feature_configuration,
action_name = C_COMPILE_ACTION_NAME,
variables = compile_variables,
)
outfile = ctx.actions.declare_file("test.o")
args = ctx.actions.args()
args.add_all(compiler_options)
ctx.actions.run(
outputs = [outfile],
inputs = ctx.files.srcs,
executable = compiler,
arguments = [args],
)
return [DefaultInfo(files = depset([outfile]))]
However, this fails with the error "execvp(external/local_config_cc/wrapped_clang, ...)": No such file or directory. I assume this is because get_tool_for_action returns a string representing a path, not a File object, so Bazel doesn't add wrapped_clang to the sandbox. Executing the rule with sandboxing disabled seems to confirm this, as it completes successfully.
Is there a way to implement this custom rule without disabling the sandbox?
If you use ctx.actions.run_shell you can add the files associated with the toolchain to the input (ctx.attr._cc_toolchain.files). Also, you'll want to add the compiler environment variables. E.g.
srcs = depset(ctx.files.srcs)
tools = ctx.attr._cc_toolchain.files
...
compiler_env = cc_common.get_environment_variables(
feature_configuration = feature_configuration,
action_name = C_COMPILE_ACTION_NAME,
variables = compiler_variables,
)
...
args = ctx.actions.args()
args.add_all(compiler_options)
ctx.actions.run_shell(
outputs = [outfile],
inputs = depset(transitive = [srcs, tools]), # Merge src and tools depsets
command = "{compiler} $*".format(compiler = compiler),
arguments = [args],
env = compiler_env,
)
Bazel doesn't add files as action inputs automatically, you have to do it explicitly, as you did in your second approach (ctx.attr._cc_toolchain.files). With that, ctx.actions.run should work just fine.
Related
I'm including a repository that has an extra_deps rule of the form:
maybe(
http_file,
name = "external_dependency",
downloaded_file_path = "foo.h",
sha256 = "<some_sha>",
urls = ["https://example.com/foo.h"],
)
If I have an existing repository, foo_repo, that provides foo.h, how can I substitute the target for it in place of external_dependency? http_file apparently provides #external_dependency//file, so I can't simply define an alias.
Using https://github.com/bazelbuild/bazel/blob/master/tools/build_defs/repo/http.bzl as a reference, you can define a custom repository rule that provides #external_dependency//file. For example:
def _repository_file(ctx):
ctx.file("WORKSPACE", "workspace(name = \"{name}\")".format(name = ctx.name))
ctx.file("file/BUILD.bazel", """
filegroup(
name = "file",
srcs = ["{}"],
visibility = ["//visibility:public"],
)
""".format(ctx.attr.source))
repository_file = repository_rule(
attrs = {"source": attr.label(mandatory = True, allow_single_file = True)},
implementation = _repository_file,
doc = """Analogue of http_file, but for a file in another repository.
Usage:
repository_file(
name = "special_file"
source = "#other_repo//path/to:special_file.txt",
)
""",
)
Now use:
repository_file(
name = "external_dependency",
source = "#foo_repo//path/to:foo.h",
)
I'm trying to define a bazel rule that will build 2 different cc_binaries for 2 different platforms with just 1 bazel build invocation. I'm struggling with how to define the transition properly and attach it.
I would like ultimately to be able to do something like:
cc_binary(
name = "binary_platform_a"
...
)
cc_binary(
name = "binary_platform_b"
...
)
my_custom_multi_arch_rule(
name = "multiarch_build",
binary_a = ":binary_platform_a",
binary_b = ":binary_platform_b",
...
)
I have deduced from bazel documents: [https://bazel.build/rules/config#user-defined-transitions] that I need to do something like the following in a defs.bzl:
def _impl(settings, attr):
_ignore = (settings, attr)
return {
"Platform A": {"//command_line_option:platform": "platform_a"},
"Platform B": {"//command_line_option:platform": "platform_b"},
}
multi_arch_transition = transition(
implementation = _impl,
inputs = [],
outputs = ["//command_line_option:platform"]
)
def _rule_impl(ctx):
# How to implement this?
my_custom_multi_arch_rule = rule(
implementation = _rule_impl,
attrs = {
"binary_a": attr.label(cfg = multi_arch_transition)
"binary_b": attr.label(cfg = multi_arch_transition)
...
})
The best-case final scenario would be able to issue:
bazel build //path/to/my:multiarch_build
and it successfully builds my 2 separate binaries for their respective platforms.
Use ctx.split_attr.<attr name>[<transition key>] to get the configured Target object representing a particular arch configuration of a binary.
def _rule_impl(ctx):
binary_a_platform_a = ctx.split_attr.binary_a["Platform A"]
binary_a_platform_b = ctx.split_attr.binary_b["Platform B"]
# ...
return [DefaultInfo(files = depset([binary_a_platform_a, binary_b_platform_b, ...]))]
https://bazel.build/rules/config#accessing-attributes-with-transitions
I am trying to write a custom rule, where I first generate a file from a template, then pass this file to a script to generate some c++ headers that are the output of my rule.
def _msg_library_impl(ctx):
# For each target in deps, print its label and files.
for source in enumerate(ctx.attr.srcs):
print("File = " + str(source))
out_header = ctx.actions.declare_file("some_header.hpp")
out_arguments = ctx.actions.declare_file("arguments.json")
ctx.actions.expand_template(
template = ctx.file._arguments_file,
output = out_arguments,
substitutions = {
"{output_dir}": out_header.dirname,
"{idl_tuples}": out_header.path,
},
)
args = ctx.actions.args()
args.add("--arguments-file")
args.add(out_arguments)
ctx.actions.run(
outputs = [out_header],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
return [
CcInfo(compilation_context=cc_common.create_compilation_context(
includes=depset([out_header.dirname]),
headers=depset([out_header])))
]
msg_library = rule(
implementation = _msg_library_impl,
output_to_genfiles = True,
attrs = {
"srcs": attr.label_list(allow_files = True),
"outs": attr.output_list(),
"_arguments_file": attr.label(
allow_single_file = [".json"],
default = Label("//examples/generation_rule:arguments_template.json"),
),
"_generator": attr.label(
default = Label("//examples/generation_rule:generator"),
executable = True,
cfg = "exec"
),
},
)
Here, generator is a python library that, given an input file provided to srcs and an arguments file generates headers.
The issue that I am facing is that it seems that the expand_template doesn't actually run before run is called, so the generated file is nowhere to be found. What am I doing wrong here? Did I misunderstand how things work?
You need to indicate the file is an input to the action, in addition to passing its path in the arguments. Change the ctx.actions.run to:
ctx.actions.run(
outputs = [out_header],
inputs = [out_arguments],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
I used a bazel macro to run a python test on a subset of source files. Similar to this:
def report(name, srcs):
source_labels = [file for file in srcs if file.startswith("a")]
if len(source_labels) == 0:
return;
source_filenames = ["$(location %s)" % x for x in source_labels]
native.py_test(
name = name + "_report",
srcs = ["report_tool"],
data = source_labels,
main = "report_tool.py",
args = source_filenames,
)
report("foo", ["foo.hpp", "afoo.hpp"])
This worked fine until one of my source files started using a select and now I get the error:
File "/home/david/foo/report.bzl", line 47, in report
[file for file in srcs if file.startswith("a")]
type 'select' is not iterable
I tried to move the code to a bazel rule, but then I get a different error that py_test can not be used in the analysis phase.
The reason that the select is causing the error is that macros are evaluated during the loading phase, whereas selectss are not evaluated until the analysis phase (see Extension Overview).
Similarly, py_test can't be used in a rule implementation because the rule implementation is evaluated in the analysis phase, whereas the py_test would need to have been loaded in the loading phase.
One way past this is to create a separate Starlark rule that takes a list of labels and just creates a file with each filename from the label. Then the py_test takes that file as data and loads the other files from there. Something like this:
def report(name, srcs):
file_locations_label = "_" + name + "_file_locations"
_generate_file_locations(
name = file_locations_label,
labels = srcs
)
native.py_test(
name = name + "_report",
srcs = ["report_tool.py"],
data = srcs + [file_locations_label],
main = "report_tool.py",
args = ["$(location %s)" % file_locations_label]
)
def _generate_file_locations_impl(ctx):
paths = []
for l in ctx.attr.labels:
f = l.files.to_list()[0]
if f.basename.startswith("a"):
paths.append(f.short_path)
ctx.actions.write(ctx.outputs.file_paths, "\n".join(paths))
return DefaultInfo(runfiles = ctx.runfiles(files = [ctx.outputs.file_paths]))
_generate_file_locations = rule(
implementation = _generate_file_locations_impl,
attrs = { "labels": attr.label_list(allow_files = True) },
outputs = { "file_paths": "%{name}_files" },
)
This has one disadvantage: Because the py_test has to depend on all the sources, the py_test will get rerun even if the only files that have changed are the ignored files. (If this is a significant drawback, then there is at least one way around this, which is to have _generate_file_locations filter the files too, and have the py_test depend on only _generate_file_locations. This could maybe be accomplished through runfiles symlinks)
Update:
Since the test report tool comes from an external repository and can't be easily modified, here's another approach that might work better. Rather than create a rule that creates a params file (a file containing the paths to process) as above, the Starlark rule can itself be a test rule that uses the report tool as the test executable:
def _report_test_impl(ctx):
filtered_srcs = []
for f in ctx.attr.srcs:
f = f.files.to_list()[0]
if f.basename.startswith("a"):
filtered_srcs.append(f)
report_tool = ctx.attr._report_test_tool
ctx.actions.write(
output = ctx.outputs.executable,
content = "{report_tool} {paths}".format(
report_tool = report_tool.files_to_run.executable.short_path,
paths = " ".join([f.short_path for f in filtered_srcs]))
)
runfiles = ctx.runfiles(files = filtered_srcs).merge(
report_tool.default_runfiles)
return DefaultInfo(runfiles = runfiles)
report_test = rule(
implementation = _report_test_impl,
attrs = {
"srcs": attr.label_list(allow_files = True),
"_report_test_tool": attr.label(default="//:report_test_tool"),
},
test = True,
)
This requires that the test report tool be a py_binary somewhere so that the test rule above can depend on it:
py_binary(
name = "report_test_tool",
srcs = ["report_tool.py"],
main = "report_tool.py",
)
I'm working on my first custom Bazel rules. The rules allow the running of bats command line tests.
I've included the rule definition below verbatim. I'm pretty happy with it so far but there's one part which feels really ugly and non-standard. If the rule user adds a binary dependency to the rule then I make sure that the binary appears on the PATH so that it can be tested. At the moment I do this by making a list of the binary paths and then appending them with $PWD which is expanded inside the script to the complete execution path. This feels hacky and error prone.
Is there a more idiomatic way to do this? I don't believe I can access the execution path in the rule due to it not being created until the execution phase.
Thanks for your help!
BATS_REPOSITORY_BUILD_FILE = """
package(default_visibility = [ "//visibility:public" ])
sh_binary(
name = "bats",
srcs = ["libexec/bats"],
data = [
"libexec/bats-exec-suite",
"libexec/bats-exec-test",
"libexec/bats-format-tap-stream",
"libexec/bats-preprocess",
],
)
"""
def bats_repositories(version="v0.4.0"):
native.new_git_repository(
name = "bats",
remote = "https://github.com/sstephenson/bats",
tag = version,
build_file_content = BATS_REPOSITORY_BUILD_FILE
)
BASH_TEMPLATE = """
#!/usr/bin/env bash
set -e
export TMPDIR="$TEST_TMPDIR"
export PATH="{bats_bins_path}":$PATH
"{bats}" "{test_paths}"
"""
def _dirname(path):
prefix, _, _ = path.rpartition("/")
return prefix.rstrip("/")
def _bats_test_impl(ctx):
runfiles = ctx.runfiles(
files = ctx.files.srcs,
collect_data = True,
)
tests = [f.short_path for f in ctx.files.srcs]
path = ["$PWD/" + _dirname(b.short_path) for b in ctx.files.deps]
sep = ctx.configuration.host_path_separator
ctx.file_action(
output = ctx.outputs.executable,
executable = True,
content = BASH_TEMPLATE.format(
bats = ctx.executable._bats.short_path,
test_paths = " ".join(tests),
bats_bins_path = sep.join(path),
),
)
runfiles = runfiles.merge(ctx.attr._bats.default_runfiles)
return DefaultInfo(
runfiles = runfiles,
)
bats_test = rule(
attrs = {
"srcs": attr.label_list(
allow_files = True,
),
"deps": attr.label_list(),
"_bats": attr.label(
default = Label("#bats//:bats"),
executable = True,
cfg = "host",
),
},
test = True,
implementation = _bats_test_impl,
)
This should be easy to support from Bazel 0.8.0 which will be released in ~2 weeks.
In your skylark implementation you should do ctx.expand_location(binary) where binary should be something like $(execpath :some-label) so you might want to just format the label you got from the user with the $(execpath) and bazel will make sure to give you the execution location of that label.
Some relevant resources:
$location expansion in Bazel
https://github.com/bazelbuild/bazel/issues/2475
https://github.com/bazelbuild/bazel/commit/cff0dc94f6a8e16492adf54c88d0b26abe903d4c