#bazel_skylib//rules:native_binary.bzl defines the native_binary rule which can be used to wrap native executables inside a bazel target. I used it to wrap a packaging tool called packfolder.exe from the Sciter SDK.
I placed the binary into my source tree at third_party/sciter/packfolder.exe and wrote this BUILD file.
# third_party/sciter/BUILD
native_binary(name = "packfolder",
src = "packfolder.exe",
out = "packfolder.exe"
)
bazel run third_party/sciter:packfolder runs with no issues. Now I want to use this target inside my custom cc_sciter_resource rule.
# third_party/sciter/sciter_rules.bzl
def _impl(ctx):
in_files = ctx.files.srcs
output_file = ctx.actions.declare_file(ctx.label.name)
ctx.actions.run(
outputs = [output_file],
inputs = in_files,
arguments = [],
executable = ctx.executable.packfolder.path)
return DefaultInfo(files = depset([output_file]))
cc_sciter_resource = rule(
implementation = _impl,
attrs = {
"srcs": attr.label_list(),
"packfolder": attr.label(
default = Label("//third_party/sciter:packfolder"),
executable = True,
cfg = "exec"
),
}
)
The trouble is, when I try to build a target that uses this rule, say
cc_sciter_resource(
name = "hello_world_resource.cpp"
srcs = [...]
)
I get the following error.
ERROR: C:/users/marki/sciter-bazel/examples/BUILD:12:19: Action examples/hello_world_resource.cpp failed (Exit -1): packfolder.exe failed: error executing command
cd C:/users/marki/_bazel_marki/kiodv2fz/execroot/sciter_bazel
bazel-out/x64_windows-opt-exec-2B5CBBC6/bin/third_party/sciter/packfolder.exe
Execution platform: #local_config_platform//:host. Note: Remote connection/protocol failed with: execution failed
Action failed to execute: java.io.IOException: ERROR: src/main/native/windows/process.cc(202): CreateProcessW("C:\users\marki\_bazel_marki\kiodv2fz\execroot\sciter_bazel\bazel-out\x64_windows-opt-exec-2B5CBBC6\bin\third_party\sciter\packfolder.exe"): The system cannot find the file specified.
(error: 2)
Target //examples:hello_world_resource.cpp failed to build
The directory C:\users\marki\_bazel_marki\kiodv2fz\execroot\sciter_bazel\bazel-out\x64_windows-opt-exec-2B5CBBC6 does not exist on my computer. So the error is accurate, but I don't know how to resolve the issue.
--- sciter_rules.bzl
+++ sciter_rules.bzl
## -6,7 +6,7 ##
outputs = [output_file],
inputs = in_files,
arguments = [],
- executable = ctx.executable.packfolder.path)
+ executable = ctx.executable.packfolder)
return DefaultInfo(files = depset([output_file]))
cc_sciter_resource = rule(
ctx.executable.packfolder.path is just a string, so Bazel doesn't know that the packfolder executable needs to be added as an input to the action.
Related
I'm creating a rules file that generate some scripts with ctx.actions.expand_template and runs these scripts with ctx.actions.run.
ctx.actions.run uses the script file generated by ctx.actions.expand_template and the 'generated file type' file(filelist file contains several file name, path information) generated from other rule files which has a dependency relationship with this rule file as input attribute.
When the script is executed in ctx.actions.run, the generated file type filelist mentioned above is not found.
If I check the sandbox path where the actual build takes place, this filelist does not exist.
What should I do?
This is a part of my rule file
def _my_rule_impl(ctx):
...
my_script = ctx.actions.declare_file("my_script.sh")
ctx.actions.expand_template(
output = compile_script,
template = ctx.file._my_template,
substitutions = {
"{TOP}": "{}".format(top_name),
"{FLISTS}": " ".join(["-f {}".format(f.short_path) for f in flists_list]),
...
},
)
compile_srcs = flists_list + srcs_list + [my_script]
outputs = ctx.outputs.executable
executable = compile_script.path
ctx.actions.run(
inputs = depset(compile_srcs),
outputs = [outputs],
executable = executable,
env = {
"HOME": "/home/grrrr",
},
)
allfiles = depset(compile_srcs)
runfiles = ctx.runfiles(files = compile_srcs)
return [DefaultInfo(
files = allfiles,
runfiles = runfiles,
)]
my_rule = rule(
implementation = _my_rule_impl,
attrs = {
"deps": attr.label_list(
mandatory = True,
),
"_my_template": attr.label(
allow_single_file = True,
default = Label("#my_rules//my_test:my_script.sh.template"),
),
...
},
executable = True,
)
As a result of checking with print, this path is the location where the script is executed.
/home/grrrr/.cache/bazel/_bazel_grrrr/.../sandbox/processwrapper-sandbox/.../execroot/my_rules/
As a result of checking with print, the script refers to sources including a filelist in this path. However, there are only source file type files. There is not a filelist.
/home/grrrr/.cache/bazel/_bazel_grrrr/.../sandbox/processwrapper-sandbox/.../execroot/my_rules/my_test
However, There is a filelist in this path. I'm wondering why this filelist is not in above directory.
/home/grrrr/.cache/bazel/_bazel_grrrr/.../sandbox/processwrapper-sandbox/.../execroot/my_rules/bazel-out/k8-fastbuild/bin/my_test
It's resolved by using sandboxfs instead of sandbox.
Here is the useful page regarding sandboxfs.
https://bazel.build/docs/sandboxing#sandboxfs
I'm trying to define a bazel rule that will build 2 different cc_binaries for 2 different platforms with just 1 bazel build invocation. I'm struggling with how to define the transition properly and attach it.
I would like ultimately to be able to do something like:
cc_binary(
name = "binary_platform_a"
...
)
cc_binary(
name = "binary_platform_b"
...
)
my_custom_multi_arch_rule(
name = "multiarch_build",
binary_a = ":binary_platform_a",
binary_b = ":binary_platform_b",
...
)
I have deduced from bazel documents: [https://bazel.build/rules/config#user-defined-transitions] that I need to do something like the following in a defs.bzl:
def _impl(settings, attr):
_ignore = (settings, attr)
return {
"Platform A": {"//command_line_option:platform": "platform_a"},
"Platform B": {"//command_line_option:platform": "platform_b"},
}
multi_arch_transition = transition(
implementation = _impl,
inputs = [],
outputs = ["//command_line_option:platform"]
)
def _rule_impl(ctx):
# How to implement this?
my_custom_multi_arch_rule = rule(
implementation = _rule_impl,
attrs = {
"binary_a": attr.label(cfg = multi_arch_transition)
"binary_b": attr.label(cfg = multi_arch_transition)
...
})
The best-case final scenario would be able to issue:
bazel build //path/to/my:multiarch_build
and it successfully builds my 2 separate binaries for their respective platforms.
Use ctx.split_attr.<attr name>[<transition key>] to get the configured Target object representing a particular arch configuration of a binary.
def _rule_impl(ctx):
binary_a_platform_a = ctx.split_attr.binary_a["Platform A"]
binary_a_platform_b = ctx.split_attr.binary_b["Platform B"]
# ...
return [DefaultInfo(files = depset([binary_a_platform_a, binary_b_platform_b, ...]))]
https://bazel.build/rules/config#accessing-attributes-with-transitions
I am trying to write a custom rule, where I first generate a file from a template, then pass this file to a script to generate some c++ headers that are the output of my rule.
def _msg_library_impl(ctx):
# For each target in deps, print its label and files.
for source in enumerate(ctx.attr.srcs):
print("File = " + str(source))
out_header = ctx.actions.declare_file("some_header.hpp")
out_arguments = ctx.actions.declare_file("arguments.json")
ctx.actions.expand_template(
template = ctx.file._arguments_file,
output = out_arguments,
substitutions = {
"{output_dir}": out_header.dirname,
"{idl_tuples}": out_header.path,
},
)
args = ctx.actions.args()
args.add("--arguments-file")
args.add(out_arguments)
ctx.actions.run(
outputs = [out_header],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
return [
CcInfo(compilation_context=cc_common.create_compilation_context(
includes=depset([out_header.dirname]),
headers=depset([out_header])))
]
msg_library = rule(
implementation = _msg_library_impl,
output_to_genfiles = True,
attrs = {
"srcs": attr.label_list(allow_files = True),
"outs": attr.output_list(),
"_arguments_file": attr.label(
allow_single_file = [".json"],
default = Label("//examples/generation_rule:arguments_template.json"),
),
"_generator": attr.label(
default = Label("//examples/generation_rule:generator"),
executable = True,
cfg = "exec"
),
},
)
Here, generator is a python library that, given an input file provided to srcs and an arguments file generates headers.
The issue that I am facing is that it seems that the expand_template doesn't actually run before run is called, so the generated file is nowhere to be found. What am I doing wrong here? Did I misunderstand how things work?
You need to indicate the file is an input to the action, in addition to passing its path in the arguments. Change the ctx.actions.run to:
ctx.actions.run(
outputs = [out_header],
inputs = [out_arguments],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
I used a bazel macro to run a python test on a subset of source files. Similar to this:
def report(name, srcs):
source_labels = [file for file in srcs if file.startswith("a")]
if len(source_labels) == 0:
return;
source_filenames = ["$(location %s)" % x for x in source_labels]
native.py_test(
name = name + "_report",
srcs = ["report_tool"],
data = source_labels,
main = "report_tool.py",
args = source_filenames,
)
report("foo", ["foo.hpp", "afoo.hpp"])
This worked fine until one of my source files started using a select and now I get the error:
File "/home/david/foo/report.bzl", line 47, in report
[file for file in srcs if file.startswith("a")]
type 'select' is not iterable
I tried to move the code to a bazel rule, but then I get a different error that py_test can not be used in the analysis phase.
The reason that the select is causing the error is that macros are evaluated during the loading phase, whereas selectss are not evaluated until the analysis phase (see Extension Overview).
Similarly, py_test can't be used in a rule implementation because the rule implementation is evaluated in the analysis phase, whereas the py_test would need to have been loaded in the loading phase.
One way past this is to create a separate Starlark rule that takes a list of labels and just creates a file with each filename from the label. Then the py_test takes that file as data and loads the other files from there. Something like this:
def report(name, srcs):
file_locations_label = "_" + name + "_file_locations"
_generate_file_locations(
name = file_locations_label,
labels = srcs
)
native.py_test(
name = name + "_report",
srcs = ["report_tool.py"],
data = srcs + [file_locations_label],
main = "report_tool.py",
args = ["$(location %s)" % file_locations_label]
)
def _generate_file_locations_impl(ctx):
paths = []
for l in ctx.attr.labels:
f = l.files.to_list()[0]
if f.basename.startswith("a"):
paths.append(f.short_path)
ctx.actions.write(ctx.outputs.file_paths, "\n".join(paths))
return DefaultInfo(runfiles = ctx.runfiles(files = [ctx.outputs.file_paths]))
_generate_file_locations = rule(
implementation = _generate_file_locations_impl,
attrs = { "labels": attr.label_list(allow_files = True) },
outputs = { "file_paths": "%{name}_files" },
)
This has one disadvantage: Because the py_test has to depend on all the sources, the py_test will get rerun even if the only files that have changed are the ignored files. (If this is a significant drawback, then there is at least one way around this, which is to have _generate_file_locations filter the files too, and have the py_test depend on only _generate_file_locations. This could maybe be accomplished through runfiles symlinks)
Update:
Since the test report tool comes from an external repository and can't be easily modified, here's another approach that might work better. Rather than create a rule that creates a params file (a file containing the paths to process) as above, the Starlark rule can itself be a test rule that uses the report tool as the test executable:
def _report_test_impl(ctx):
filtered_srcs = []
for f in ctx.attr.srcs:
f = f.files.to_list()[0]
if f.basename.startswith("a"):
filtered_srcs.append(f)
report_tool = ctx.attr._report_test_tool
ctx.actions.write(
output = ctx.outputs.executable,
content = "{report_tool} {paths}".format(
report_tool = report_tool.files_to_run.executable.short_path,
paths = " ".join([f.short_path for f in filtered_srcs]))
)
runfiles = ctx.runfiles(files = filtered_srcs).merge(
report_tool.default_runfiles)
return DefaultInfo(runfiles = runfiles)
report_test = rule(
implementation = _report_test_impl,
attrs = {
"srcs": attr.label_list(allow_files = True),
"_report_test_tool": attr.label(default="//:report_test_tool"),
},
test = True,
)
This requires that the test report tool be a py_binary somewhere so that the test rule above can depend on it:
py_binary(
name = "report_test_tool",
srcs = ["report_tool.py"],
main = "report_tool.py",
)
I'm working on my first custom Bazel rules. The rules allow the running of bats command line tests.
I've included the rule definition below verbatim. I'm pretty happy with it so far but there's one part which feels really ugly and non-standard. If the rule user adds a binary dependency to the rule then I make sure that the binary appears on the PATH so that it can be tested. At the moment I do this by making a list of the binary paths and then appending them with $PWD which is expanded inside the script to the complete execution path. This feels hacky and error prone.
Is there a more idiomatic way to do this? I don't believe I can access the execution path in the rule due to it not being created until the execution phase.
Thanks for your help!
BATS_REPOSITORY_BUILD_FILE = """
package(default_visibility = [ "//visibility:public" ])
sh_binary(
name = "bats",
srcs = ["libexec/bats"],
data = [
"libexec/bats-exec-suite",
"libexec/bats-exec-test",
"libexec/bats-format-tap-stream",
"libexec/bats-preprocess",
],
)
"""
def bats_repositories(version="v0.4.0"):
native.new_git_repository(
name = "bats",
remote = "https://github.com/sstephenson/bats",
tag = version,
build_file_content = BATS_REPOSITORY_BUILD_FILE
)
BASH_TEMPLATE = """
#!/usr/bin/env bash
set -e
export TMPDIR="$TEST_TMPDIR"
export PATH="{bats_bins_path}":$PATH
"{bats}" "{test_paths}"
"""
def _dirname(path):
prefix, _, _ = path.rpartition("/")
return prefix.rstrip("/")
def _bats_test_impl(ctx):
runfiles = ctx.runfiles(
files = ctx.files.srcs,
collect_data = True,
)
tests = [f.short_path for f in ctx.files.srcs]
path = ["$PWD/" + _dirname(b.short_path) for b in ctx.files.deps]
sep = ctx.configuration.host_path_separator
ctx.file_action(
output = ctx.outputs.executable,
executable = True,
content = BASH_TEMPLATE.format(
bats = ctx.executable._bats.short_path,
test_paths = " ".join(tests),
bats_bins_path = sep.join(path),
),
)
runfiles = runfiles.merge(ctx.attr._bats.default_runfiles)
return DefaultInfo(
runfiles = runfiles,
)
bats_test = rule(
attrs = {
"srcs": attr.label_list(
allow_files = True,
),
"deps": attr.label_list(),
"_bats": attr.label(
default = Label("#bats//:bats"),
executable = True,
cfg = "host",
),
},
test = True,
implementation = _bats_test_impl,
)
This should be easy to support from Bazel 0.8.0 which will be released in ~2 weeks.
In your skylark implementation you should do ctx.expand_location(binary) where binary should be something like $(execpath :some-label) so you might want to just format the label you got from the user with the $(execpath) and bazel will make sure to give you the execution location of that label.
Some relevant resources:
$location expansion in Bazel
https://github.com/bazelbuild/bazel/issues/2475
https://github.com/bazelbuild/bazel/commit/cff0dc94f6a8e16492adf54c88d0b26abe903d4c