How bazel genrule srcs works under the hood - bazel

Well, I'm trying to figure out how srcs works for genrule, I have the following:
genrule(
name = "flutter_build_android",
srcs = [
"//:genfiles"
],
outs = ["android/mobile.apk"],
cmd_bash = "ls -ltr && flutter build apk > $#",
tags = ["local"]
)
//:genfiles is a filegroup using glob:
filegroup(
name = "genfiles",
srcs = glob(["lib/**","assets/**", "pubspec.yaml"])
)
When executing my genrule what I expect is only files under ://genfiles label should be returned, but it is returning all folders under my root project:

What's happening is that the genrule is tagged with local, and local means
precludes the action or test from being remotely cached, remotely executed, or run inside the sandbox
https://bazel.build/reference/be/common-definitions#common.tags
Sandboxing is what prevents an action from seeing files that aren't declared as dependencies (i.e. in srcs here). Without the sandbox, the action sees everything.
$ tree
.
├── BUILD
├── file1
├── file2
├── file3
└── WORKSPACE
0 directories, 5 files
$ cat BUILD
genrule(
name = "gen_foo",
outs = ["foo"],
srcs = ["file1"],
cmd = "echo ----- ; ls ; echo ----- ; wc -l $< > $#",
# tags = ["local"],
)
$ bazel build foo
INFO: Analyzed target //:foo (5 packages loaded, 9 targets configured).
INFO: Found 1 target...
INFO: From Executing genrule //:gen_foo:
-----
bazel-out
external
file1
-----
Target //:foo up-to-date:
bazel-bin/foo
INFO: Elapsed time: 0.379s, Critical Path: 0.02s
INFO: 2 processes: 1 internal, 1 linux-sandbox.
INFO: Build completed successfully, 2 total actions
# edit BUILD to restore the local tag
$ cat BUILD
genrule(
name = "gen_foo",
outs = ["foo"],
srcs = ["file1"],
cmd = "echo ----- ; ls ; echo ----- ; wc -l $< > $#",
tags = ["local"],
)
$ bazel build foo
INFO: Analyzed target //:foo (5 packages loaded, 9 targets configured).
INFO: Found 1 target...
INFO: From Executing genrule //:gen_foo:
-----
BUILD
WORKSPACE
bazel-out
external
file1
file2
file3
local-spawn-runner.8714966150718292736
-----
Target //:foo up-to-date:
bazel-bin/foo
INFO: Elapsed time: 0.350s, Critical Path: 0.02s
INFO: 2 processes: 1 internal, 1 local.
INFO: Build completed successfully, 2 total actions

Related

Erlang Hello World using Bazel as a build system

I want to build an Erlang hello world example using rules_erlang on Ubuntu 22.04.
My setup looks like this:
BUILD.bazel
load("#rules_erlang//:erlang_app.bzl", "erlang_app", "test_erlang_app")
load("#rules_erlang//:xref.bzl", "xref")
load("#rules_erlang//:dialyze.bzl", "dialyze", "plt")
load("#rules_erlang//:ct.bzl", "ct_suite", "assert_suites")
APP_NAME = "hello_world"
APP_VERSION = "0.1.0"
erlang_app(
app_name = APP_NAME,
app_version = APP_VERSION,
)
src/hello_world.erl
-module(hello_world).
-compile(export_all).
hello() ->
io:format("hello world~n").
WORKSPACE.bazel
load("#bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "bazel_skylib",
sha256 = "af87959afe497dc8dfd4c6cb66e1279cb98ccc84284619ebfec27d9c09a903de",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
],
)
load("#bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
bazel_skylib_workspace()
http_archive(
name = "rules_erlang",
sha256 = "5e59800ecc786d5375951028c959c6e6275c94eff2a52f5d53ccb1ad8b2ea20a",
strip_prefix = "rules_erlang-3.8.4",
urls = ["https://github.com/rabbitmq/rules_erlang/archive/refs/tags/3.8.4.zip"],
)
load(
"#rules_erlang//:rules_erlang.bzl",
"erlang_config",
"rules_erlang_dependencies",
)
erlang_config()
rules_erlang_dependencies()
load("#erlang_config//:defaults.bzl", "register_defaults")
register_defaults()
The code can also found here.
When I execute bazel build //... I get
ERROR:
/home/vertexwahn/.cache/bazel/_bazel_vertexwahn/b5f945f94177a8ffa6ac0f7108dfc1cd/external/erlang_config/external/BUILD.bazel:12:16:
Validating otp at /usr failed: (Exit 1): bash failed: error executing
command /bin/bash -c ... (remaining 1 argument skipped)
Use --sandbox_debug to see verbose messages from the sandbox and
retain the sandbox build root for debugging Erlang version mismatch
(Expected UNKNOWN, found 24.2.1)
Any hints to get this working are welcome!
To run it, you can declare a shell rule in your BUILD.bazel, for instance:
load("#rules_erlang//:shell.bzl", "shell")
shell(
name = "repl",
deps = [":erlang_app"],
extra_erl_args = ["-eval", "'hello_world:hello()'"],
)
and then you could bazel run :repl.
Or, you could use an escript rule if you change hello_world.erl so that it exports main/1 instead of hello:
load("#rules_erlang//:escript.bzl", "escript_archive")
escript_archive(
name = "hello_world",
app = ":erlang_app",
)
hello_world.erl:
-module(hello_world).
-export([main/1]).
main(_) ->
io:format("hello world~n").
and run with bazel run :hello_world
bazel build //... --sandbox_debug
gave me
compile: warnings being treated as errors
hello_world.erl:2:2: export_all flag enabled - all functions will be exported
With -export([hello/0]). instead of -compile(export_all). it works
-module(hello_world).
-export([hello/0]).
hello() ->
io:format("hello world~n").
❯ bazel build //...
INFO: Analyzed 3 targets (0 packages loaded, 0 targets configured).
INFO: Found 3 targets...
INFO: Elapsed time: 0,326s, Critical Path: 0,25s
INFO: 2 processes: 1 internal, 1 darwin-sandbox.
INFO: Build completed successfully, 2 total actions

Why doesn't this rule create a file every time it is built?

I write the below bazel rule for incremental test.
This rule(foo_library) has a dependency property.
When there are no dependencies, it creates a file with its own name, filling the contents of the file with its own name.
When there is a dependency, it creates a file with its own name, and at this time, the contents of the file are filled by adding its own name to the contents of the file created by the dependency target.
In this test, foo3, foo2, and foo1 are linked as dependencies, so I expected 3 files to be created, but only foo3 was created.
What could be the problem?
Hera are my test bazel code, build command and log message.
# ~/rules/dev.bzl
def _foo_binary_impl(ctx):
this_name = ctx.attr.name
file_name = ctx.actions.declare_file(ctx.label.name)
print("##-000 this_name: {}".format(this_name))
print("##-001 file_name: {}".format(file_name))
file_content = ""
if ctx.attr.deps:
for dep in ctx.attr.deps:
t1 = dep[DefaultInfo].files.to_list()[0].basename
t1 = t1 + this_name
print("##-100 t1: {}".format(t1))
file_content = t1
else:
t2 = this_name
print("##-101 t2: {}".format(t2))
file_content = t2
ctx.actions.write(
output = file_name,
content = file_content,
)
return [
DefaultInfo(
files = depset([file_name])
),
]
foo_binary = rule(
implementation = _foo_binary_impl,
attrs = {
"deps": attr.label_list(),
},
)
# build command and log
$> bazel build -s //rules/example:foo3 1 х 11:38:08
DEBUG: ~/rules/dev.bzl:10:10: ##-000 this_name: foo1
DEBUG: ~/rules/dev.bzl:11:10: ##-001 file_name: <generated file rules/example/foo1>
DEBUG: ~/rules/dev.bzl:25:14: ##-101 t2: foo1
DEBUG: ~/rules/dev.bzl:10:10: ##-000 this_name: foo2
DEBUG: ~/rules/dev.bzl:11:10: ##-001 file_name: <generated file rules/example/foo2>
DEBUG: ~/rules/dev.bzl:19:18: ##-100 t1: foo1foo2
DEBUG: ~/rules/dev.bzl:10:10: ##-000 this_name: foo3
DEBUG: ~/rules/dev.bzl:11:10: ##-001 file_name: <generated file rules/example/foo3>
DEBUG: ~/rules/dev.bzl:19:18: ##-100 t1: foo2foo3
INFO: Analyzed target //rules/example:foo3 (3 packages loaded, 8 targets configured).
INFO: Found 1 target...
Target //rules/example:foo3 up-to-date:
bazel-bin/rules/example/foo3
INFO: Elapsed time: 0.107s, Critical Path: 0.00s
INFO: 2 processes: 2 internal.
INFO: Build completed successfully, 2 total actions
Build foo3 and expected 3 files(foo1, foo2, foo3) are generated. howver, foo3 is generated only.
If ctx.attr.deps is nonempty, foo_binary_impl writes a file with content that depends on the file name of the last file in ctx.attr.deps rather than the content of the last file in ctx.attr.deps. No dependency on any file in ctx.attr is ever introduced. Therefore, Bazel does not build anything in deps when producing the output of foo_binary_impl.
To make the output of foo_binary_impl depend on the content of dependencies, make an action to produce the final output that takes the dependency files as inputs. For example, to concatenate all the files in deps into the output:
ctx.actions.run_shell(
inputs = ctx.files.deps,
outputs = [file_name],
command = "cat " + " ".join([f.path for f in ctx.files.deps]) + " >" + file_name.path,
)

get variable from config_setting in bazel

I have the following config_setting defined:
config_setting(
name = "perception_env",
values = {"perception": "true"},
)
print(perception_env)
However, I can't seem to print the variable, it says it doesn't exist.
config_setting is only used for selecting the different possible values in a select(). A config_setting doesn't really have a value, it's more an association of a variable (a Bazel flag, a Starlark-defined flag, platform constraints) and its value. The values attribute is basically for flag values ("perception" would have to be a bazel flag).
For example,
config_setting(
name = "my_config_setting_opt",
values = {"compilation_mode": "opt"}
)
config_setting(
name = "config_setting_dbg",
values = {"compilation_mode": "dbg"}
)
config_setting(
name = "config_setting_fastbuild",
values = {"compilation_mode": "fastbuild"}
)
genrule(
name = "gen_out",
outs = ["out"],
cmd = select({
":my_config_setting_opt": "echo Opt mode > $#",
":config_setting_dbg": "echo Dbg mode > $#",
":config_setting_fastbuild": "echo Fastbuild mode > $#",
}),
)
The 3 config_settings declare 3 different associations of the --compilation_mode flag, one for each of its possible values (see https://bazel.build/docs/user-manual#compilation-mode)
Then the select() declares 3 different possible values for the cmd attribute of the genrule gen_out. Then setting the --compilation_mode flag to different values changes which value for cmd is selected:
$ bazel build out --compilation_mode=dbg && cat bazel-bin/out
INFO: Build option --compilation_mode has changed, discarding analysis cache.
INFO: Analyzed target //:out (0 packages loaded, 11 targets configured).
INFO: Found 1 target...
Target //:out up-to-date:
bazel-bin/out
INFO: Elapsed time: 0.145s, Critical Path: 0.01s
INFO: 2 processes: 1 internal, 1 linux-sandbox.
INFO: Build completed successfully, 2 total actions
Dbg mode
$ bazel build out --compilation_mode=opt && cat bazel-bin/out
INFO: Build option --compilation_mode has changed, discarding analysis cache.
INFO: Analyzed target //:out (0 packages loaded, 11 targets configured).
INFO: Found 1 target...
Target //:out up-to-date:
bazel-bin/out
INFO: Elapsed time: 0.111s, Critical Path: 0.01s
INFO: 2 processes: 1 internal, 1 linux-sandbox.
INFO: Build completed successfully, 2 total actions
Opt mode
$ bazel build out --compilation_mode=fastbuild && cat bazel-bin/out
INFO: Build option --compilation_mode has changed, discarding analysis cache.
INFO: Analyzed target //:out (0 packages loaded, 11 targets configured).
INFO: Found 1 target...
Target //:out up-to-date:
bazel-bin/out
INFO: Elapsed time: 0.145s, Critical Path: 0.01s
INFO: 1 process: 1 internal.
INFO: Build completed successfully, 1 total action
Fastbuild mode

Bazel: why lint failed for variable reference?

I had this genrule in BUILD file, but bazel build failed for the error of:
in cmd attribute of genrule rule //example:create_version_pom: $(BUILD_TAG) not defined
genrule(
name = "create_version_pom",
srcs = ["pom_template.xml"],
outs = ["version_pom.xml"],
cmd = "sed 's/BUILD_TAG/$(BUILD_TAG)/g' $< > $#",
)
What's the reason, and how to fix it please?
The cmd attribute of genrule will do variable expansion for Bazel build variables before the command is executed. The $< and $# variables for the input file and the output file are some of the pre-defined variables. Variables can be defined with --define, e.g.:
$ cat BUILD
genrule(
name = "genfoo",
outs = ["foo"],
cmd = "echo $(bar) > $#",
)
$ bazel build foo --define=bar=123
INFO: Analyzed target //:foo (5 packages loaded, 8 targets configured).
INFO: Found 1 target...
Target //:foo up-to-date:
bazel-bin/foo
INFO: Elapsed time: 0.310s, Critical Path: 0.01s
INFO: 2 processes: 1 internal, 1 linux-sandbox.
INFO: Build completed successfully, 2 total actions
$ cat bazel-bin/foo
123
So to have $(BUILD_TAG) work in the genrule, you'll want to pass
--define=BUILD_TAG=the_build_tag
Unless it's that you want BUILD_TAG replaced with literally $(BUILD_TAG), in which case the $ needs to be escaped with another $: $$(BUILD_TAG).
See
https://docs.bazel.build/versions/main/be/general.html#genrule.cmd
https://docs.bazel.build/versions/main/be/make-variables.html
Note that Bazel also has a mechanism for "build stamping" for bringing information like build time and version numbers into the build:
https://docs.bazel.build/versions/main/user-manual.html#workspace_status
https://docs.bazel.build/versions/main/command-line-reference.html#flag--embed_label
Using --workspace_status_command and --embed_label are a little more complicated though.

How to see Bazel build output?

How do you view stdout of bazel build as it happens?
I want to see all the logs written to stdout during a bazel build.
No one of these allows it to show the ls command before after it has failed
$ bazel build --show_progress --worker_verbose --verbose_failures --verbose_explanations=true -s --test_output=streamed :build
genrule(
name = "build",
cmd = "ls && sleep 60 && exit 1",
)
$ bazel build --show_progress --worker_verbose --verbose_failures --verbose_explanations=true -s --test_output=streamed :build
WARNING: --verbose_explanations has no effect when --explain=<file> is not enabled
INFO: Analyzed target //:build (0 packages loaded, 0 targets configured).
INFO: Found 1 target...
SUBCOMMAND: # //:build [action 'Executing genrule //:build']
(cd /private/var/tmp/_bazel_kevinsimper/f9e6a72c146c5ad83b84a8ebf539f8b2/execroot/__main__ && \
exec env - \
PATH=/usr/local/sbin \
/bin/bash -c 'source external/bazel_tools/tools/genrule/genrule-setup.sh; ls && sleep 60 && exit 1')
ERROR: /Users/kevinsimper/testproject/BUILD:1:1: Executing genrule //:build failed (Exit 1)
BUILD
TESTFILE
Target //:build failed to build
INFO: Elapsed time: 60.256s, Critical Path: 60.04s
INFO: 0 processes.
FAILED: Build did NOT complete successfully
There's no way to stream action stdout/stderr while it's executing, unless it's a test while using the --test_output=streamed flag.
You can follow an underhanded approach. If your build includes small number of lengthy actions, it is feasible to snoop for output in bazel-out/_tmp/actions/std{err,out}-* as it happens. This works for me with Bazel 3.1.0.

Resources