I've tried to convert a bazel macro to a rule, so it's parsing is done now on analysis time rather than on loading time, this makes calling native.cc_binary impossible to do
def _emcc_binary(ctx):
includejs = False
includehtml = False
linkopts = list(ctx.attr.linkopts)
linkopts.append("-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]'")
if ctx.attr.name.endswith(".html"):
basename = ctx.attr.name[:-5]
includehtml = True
includejs = True
elif ctx.attr.name.endswith(".js"):
basename = ctx.attr.name[:-3]
includejs = True
outputs = []
if includejs:
outputs.append(basename + ".js")
if ctx.attr.wasm:
outputs.append(basename + ".wasm")
if ctx.attr.memory_init_file:
outputs.append(basename + ".mem")
if ctx.attr.worker:
outputs.append(basename + ".worker.js")
linkopts.append("--proxy-to-worker")
if includehtml:
outputs.append(basename + ".html")
if not ctx.attr.wasm:
linkopts.append("-s WASM=0")
linkopts.append("--memory-init-file %d" % ctx.attr.memory_init_file)
if includejs:
tarfile = ctx.attr.name + ".tar"
# we'll generate a tarfile and extract multiple outputs
native.cc_binary(name = tarfile, linkopts = linkopts, **ctx.attr.kwargs)
native.genrule(
name = "emcc_extract_" + tarfile,
srcs = [tarfile],
outs = outputs,
output_to_bindir = 1,
testonly = ctx.attr.kwargs.get("testonly"),
cmd = """
tar xvf $< -C "$(#D)"/$$(dirname "%s")
""" % [outputs[0]],
)
else:
native.cc_binary(name = ctx.attr.name, linkopts = linkopts, **ctx.attr.kwargs)
# we'll generate a tarfile and extract multiple outputs
emcc_binary = rule(
implementation = _emcc_binary,
attrs = {
"memory_init_file": attr.int(default = 0),
"wasm": attr.bool(default = True),
"worker": attr.bool(default = False),
"srcs": attr.label_list(allow_files = True),
"linkopts": attr.string_list(),
"noop": attr.bool(default = False),
"kwargs": attr.label_keyed_string_dict()
},
)
output:
cc_binary() cannot be called during the analysis phase
ERROR: Analysis of target '//tests:hi.js' failed; build aborted: Analysis of target '//tests:hi.js' failed; build aborted
Correct, rules cannot be called from the implementations of other rules. Typically the solution is to use both rules and macros, where the macro creates and wires up the other rules (including your custom Starlark rules) where necessary.
Looking over your code, it's not clear that you really need to change the macro to a rule. Might be helpful if you could describe your end goal (feel free to reach out to bazel-discuss if you'd like more help here). That said, if you really wanted, you can replace native.genrule with actions and native.cc_binary with cc_common, which are designed to work with rules.
Related
I am trying to write a custom rule, where I first generate a file from a template, then pass this file to a script to generate some c++ headers that are the output of my rule.
def _msg_library_impl(ctx):
# For each target in deps, print its label and files.
for source in enumerate(ctx.attr.srcs):
print("File = " + str(source))
out_header = ctx.actions.declare_file("some_header.hpp")
out_arguments = ctx.actions.declare_file("arguments.json")
ctx.actions.expand_template(
template = ctx.file._arguments_file,
output = out_arguments,
substitutions = {
"{output_dir}": out_header.dirname,
"{idl_tuples}": out_header.path,
},
)
args = ctx.actions.args()
args.add("--arguments-file")
args.add(out_arguments)
ctx.actions.run(
outputs = [out_header],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
return [
CcInfo(compilation_context=cc_common.create_compilation_context(
includes=depset([out_header.dirname]),
headers=depset([out_header])))
]
msg_library = rule(
implementation = _msg_library_impl,
output_to_genfiles = True,
attrs = {
"srcs": attr.label_list(allow_files = True),
"outs": attr.output_list(),
"_arguments_file": attr.label(
allow_single_file = [".json"],
default = Label("//examples/generation_rule:arguments_template.json"),
),
"_generator": attr.label(
default = Label("//examples/generation_rule:generator"),
executable = True,
cfg = "exec"
),
},
)
Here, generator is a python library that, given an input file provided to srcs and an arguments file generates headers.
The issue that I am facing is that it seems that the expand_template doesn't actually run before run is called, so the generated file is nowhere to be found. What am I doing wrong here? Did I misunderstand how things work?
You need to indicate the file is an input to the action, in addition to passing its path in the arguments. Change the ctx.actions.run to:
ctx.actions.run(
outputs = [out_header],
inputs = [out_arguments],
progress_message = "Generating headers '{}'".format(out_header.short_path),
executable = ctx.executable._generator,
arguments = [args],
)
I'm trying to write a macro to abstract over a bunch of rules that I need for a couple of different targets. One thing that I need to do is to create a small file that will be treated as a source file in a later rule. If this was a rule I would just use expand_template. The best I can currently come up with involves a native.genrule and making sure I escape everything correctly and passing it to echo.
I would hope there is an easier way.
Code in question:
racket_contents = """
#lang racket/base
(require
"bootstrap-compiler.rkt"
racket/runtime-path)
(define-runtime-path library-compiler-list-file "%s")
(run-bootstrap-compiler library-compiler-list-file #"%s_main")
""" % (source_file_list, short_name)
native.genrule(
name = "racketize_" + name,
outs = [racket_src_name],
cmd = "echo >>$# '%s'" % racket_contents,
)
You can have a generic rule for expanding templates.
genfile.bzl
def _genfile_impl(ctx):
ctx.actions.expand_template(
template = ctx.file.template,
output = ctx.file.output,
substitutions = substitutions,
)
return [
DefaultInfo(files = depset([ctx.file.output])),
]
genfile = rule(
implementation = _genfile_impl,
attrs = {
"template": attr.label(
mandatory = True,
allow_single_file = True,
),
"output": attr.output(
mandatory = True,
),
"substitutions": attr.string_dict(),
},
)
racket.tpl
#lang racket/base
(require
"bootstrap-compiler.rkt"
racket/runtime-path)
(define-runtime-path library-compiler-list-file "TMPL_SOURCE_FILES")
(run-bootstrap-compiler library-compiler-list-file #"TMPL_SHORTNAME_main")
BUILD
load("//:genfile.bzl", "genfile")
genfile(
name = "racket",
output = "racket.out",
template = "racket.tpl",
substitutions = {
"TMPL_SOURCE_FILES": ",",join(["n1","n2"]),
"TMPL_SHORTNAME": "shortname",
},
)
I used a bazel macro to run a python test on a subset of source files. Similar to this:
def report(name, srcs):
source_labels = [file for file in srcs if file.startswith("a")]
if len(source_labels) == 0:
return;
source_filenames = ["$(location %s)" % x for x in source_labels]
native.py_test(
name = name + "_report",
srcs = ["report_tool"],
data = source_labels,
main = "report_tool.py",
args = source_filenames,
)
report("foo", ["foo.hpp", "afoo.hpp"])
This worked fine until one of my source files started using a select and now I get the error:
File "/home/david/foo/report.bzl", line 47, in report
[file for file in srcs if file.startswith("a")]
type 'select' is not iterable
I tried to move the code to a bazel rule, but then I get a different error that py_test can not be used in the analysis phase.
The reason that the select is causing the error is that macros are evaluated during the loading phase, whereas selectss are not evaluated until the analysis phase (see Extension Overview).
Similarly, py_test can't be used in a rule implementation because the rule implementation is evaluated in the analysis phase, whereas the py_test would need to have been loaded in the loading phase.
One way past this is to create a separate Starlark rule that takes a list of labels and just creates a file with each filename from the label. Then the py_test takes that file as data and loads the other files from there. Something like this:
def report(name, srcs):
file_locations_label = "_" + name + "_file_locations"
_generate_file_locations(
name = file_locations_label,
labels = srcs
)
native.py_test(
name = name + "_report",
srcs = ["report_tool.py"],
data = srcs + [file_locations_label],
main = "report_tool.py",
args = ["$(location %s)" % file_locations_label]
)
def _generate_file_locations_impl(ctx):
paths = []
for l in ctx.attr.labels:
f = l.files.to_list()[0]
if f.basename.startswith("a"):
paths.append(f.short_path)
ctx.actions.write(ctx.outputs.file_paths, "\n".join(paths))
return DefaultInfo(runfiles = ctx.runfiles(files = [ctx.outputs.file_paths]))
_generate_file_locations = rule(
implementation = _generate_file_locations_impl,
attrs = { "labels": attr.label_list(allow_files = True) },
outputs = { "file_paths": "%{name}_files" },
)
This has one disadvantage: Because the py_test has to depend on all the sources, the py_test will get rerun even if the only files that have changed are the ignored files. (If this is a significant drawback, then there is at least one way around this, which is to have _generate_file_locations filter the files too, and have the py_test depend on only _generate_file_locations. This could maybe be accomplished through runfiles symlinks)
Update:
Since the test report tool comes from an external repository and can't be easily modified, here's another approach that might work better. Rather than create a rule that creates a params file (a file containing the paths to process) as above, the Starlark rule can itself be a test rule that uses the report tool as the test executable:
def _report_test_impl(ctx):
filtered_srcs = []
for f in ctx.attr.srcs:
f = f.files.to_list()[0]
if f.basename.startswith("a"):
filtered_srcs.append(f)
report_tool = ctx.attr._report_test_tool
ctx.actions.write(
output = ctx.outputs.executable,
content = "{report_tool} {paths}".format(
report_tool = report_tool.files_to_run.executable.short_path,
paths = " ".join([f.short_path for f in filtered_srcs]))
)
runfiles = ctx.runfiles(files = filtered_srcs).merge(
report_tool.default_runfiles)
return DefaultInfo(runfiles = runfiles)
report_test = rule(
implementation = _report_test_impl,
attrs = {
"srcs": attr.label_list(allow_files = True),
"_report_test_tool": attr.label(default="//:report_test_tool"),
},
test = True,
)
This requires that the test report tool be a py_binary somewhere so that the test rule above can depend on it:
py_binary(
name = "report_test_tool",
srcs = ["report_tool.py"],
main = "report_tool.py",
)
I'm working on my first custom Bazel rules. The rules allow the running of bats command line tests.
I've included the rule definition below verbatim. I'm pretty happy with it so far but there's one part which feels really ugly and non-standard. If the rule user adds a binary dependency to the rule then I make sure that the binary appears on the PATH so that it can be tested. At the moment I do this by making a list of the binary paths and then appending them with $PWD which is expanded inside the script to the complete execution path. This feels hacky and error prone.
Is there a more idiomatic way to do this? I don't believe I can access the execution path in the rule due to it not being created until the execution phase.
Thanks for your help!
BATS_REPOSITORY_BUILD_FILE = """
package(default_visibility = [ "//visibility:public" ])
sh_binary(
name = "bats",
srcs = ["libexec/bats"],
data = [
"libexec/bats-exec-suite",
"libexec/bats-exec-test",
"libexec/bats-format-tap-stream",
"libexec/bats-preprocess",
],
)
"""
def bats_repositories(version="v0.4.0"):
native.new_git_repository(
name = "bats",
remote = "https://github.com/sstephenson/bats",
tag = version,
build_file_content = BATS_REPOSITORY_BUILD_FILE
)
BASH_TEMPLATE = """
#!/usr/bin/env bash
set -e
export TMPDIR="$TEST_TMPDIR"
export PATH="{bats_bins_path}":$PATH
"{bats}" "{test_paths}"
"""
def _dirname(path):
prefix, _, _ = path.rpartition("/")
return prefix.rstrip("/")
def _bats_test_impl(ctx):
runfiles = ctx.runfiles(
files = ctx.files.srcs,
collect_data = True,
)
tests = [f.short_path for f in ctx.files.srcs]
path = ["$PWD/" + _dirname(b.short_path) for b in ctx.files.deps]
sep = ctx.configuration.host_path_separator
ctx.file_action(
output = ctx.outputs.executable,
executable = True,
content = BASH_TEMPLATE.format(
bats = ctx.executable._bats.short_path,
test_paths = " ".join(tests),
bats_bins_path = sep.join(path),
),
)
runfiles = runfiles.merge(ctx.attr._bats.default_runfiles)
return DefaultInfo(
runfiles = runfiles,
)
bats_test = rule(
attrs = {
"srcs": attr.label_list(
allow_files = True,
),
"deps": attr.label_list(),
"_bats": attr.label(
default = Label("#bats//:bats"),
executable = True,
cfg = "host",
),
},
test = True,
implementation = _bats_test_impl,
)
This should be easy to support from Bazel 0.8.0 which will be released in ~2 weeks.
In your skylark implementation you should do ctx.expand_location(binary) where binary should be something like $(execpath :some-label) so you might want to just format the label you got from the user with the $(execpath) and bazel will make sure to give you the execution location of that label.
Some relevant resources:
$location expansion in Bazel
https://github.com/bazelbuild/bazel/issues/2475
https://github.com/bazelbuild/bazel/commit/cff0dc94f6a8e16492adf54c88d0b26abe903d4c
What I figured currently is creating a AllTest and run it with junit. But, I am not satisfied with it. I want this rule can create as many tests as many java test file in created in codebase.
def junit_suite_test(name, srcs, deps, size="small", resources=[], classpath_resources=[], jvm_flags=[], tags=[], data=[]):
tests = []
package = PACKAGE_NAME.replace("src/test/java/", "").replace("/", ".")
for src in srcs:
if src.endswith("Test.java"):
if "/" in src:
src = package + "." + src.replace("/", ".")
tests += [src.replace(".java", ".class")]
native.genrule(
name = name + "-AllTests-gen",
outs = ["AllTests.java"],
cmd = """
cat <<EOF >> $#
package %s;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
#RunWith(Suite.class)
#Suite.SuiteClasses({%s})
public class AllTests {}
EOF
""" % (package, ",".join(tests))
)
native.java_test(
name = name,
srcs = srcs + ["AllTests.java"],
test_class = package + ".AllTests",
resources = resources,
classpath_resources = classpath_resources,
data = data,
size = size,
tags = tags,
jvm_flags = jvm_flags,
deps = deps + [
],
)
Hi you can do something like that:
[java_test(name = s[:-5], srcs = s) for s in glob(["*.java"])]
That will create on test target per java file.
With that method, your macro would looks like:
def junit_suite_test(name, srcs, deps, size="small", resources=[], classpath_resources=[], jvm_flags=[], tags=[], data=[]):
[native.java_test(
name = name,
srcs = src,
resources = resources,
classpath_resources = classpath_resources,
data = data,
size = size,
tags = tags,
jvm_flags = jvm_flags,
deps = deps,
) for src in srcs if src.endswith("Test.java")]
Of course you probably needs some adaptation to feed in the good sources.
However, I would recommend against doing that over your solution as too much parallelisms can actually be slower in fine. The test log and the XML file will report the actual failing test case and you can use shard_count to increase parallelism is really needed.