Convert http_archive to repository_rule - bazel

I'm trying to convert a binary fetching from http_archive:
def purescript_toolchain():
http_archive(
name = "purs",
urls = ["https://github.com/purescript/purescript/releases/download/v0.14.7/linux64.tar.gz"],
sha256 = "cae16a0017c63fd83e029ca5a01cb9fc02cacdbd805b1d2b248f9bb3c3ea926d",
strip_prefix = strip_"purescript",
build_file_content = """exports_files(["purs"])""",
)
To a repository rule based approach:
def _purescript_toolchain_impl(repository_ctx):
repository_ctx.download_and_extract(
url = "https://github.com/purescript/purescript/releases/download/v0.14.7/linux64.tar.gz",
sha256 = "cae16a0017c63fd83e029ca5a01cb9fc02cacdbd805b1d2b248f9bb3c3ea926d",
stripPrefix = "purescript",
)
repository_ctx.file(
"BUILD.bazel",
content = """exports_files(["purs"])""",
)
_purescript_toolchain = repository_rule(
_purescript_toolchain_impl,
configure = True,
environ = ["PATH"],
)
def purescript_toolchain():
_purescript_toolchain(name = "purs")
It seems to work, but when I try to call other functions:
run_template = """
#!/usr/bin/env bash
set -o errexit
node -e "require('./{target_path}/{entry_module}/index.js').{entry_function}({entry_params})"
"""
def _purescript_compile(ctx):
srcs = ctx.files.srcs + ctx.files.deps
target = ctx.actions.declare_file(ctx.outputs.target.basename)
purs = ctx.executable.purs
flags = " ".join(ctx.attr.compiler_flags)
bazel_ps_deps = []
for d in ctx.attr.deps:
for f in d.files.to_list():
if f.basename == "target_srcs":
bazel_ps_deps = [f.path + "/**/*.purs"] + bazel_ps_deps
compileCmd = "\n".join(
[ "set -o errexit"
, """mkdir "$2" """
, """ "$1" compile """ + flags + """ --output "$2" "${#:3}" """
]
)
ctx.actions.run_shell(
tools = srcs + [purs],
outputs = [target],
command = compileCmd,
arguments = [purs.path, target.path] +
[src.path for src in srcs if src.extension == "purs"] +
bazel_ps_deps,
)
cpSrcsCmd = "\n".join(
[ "set -o errexit"
, """mkdir -p "$1" """
, """cp "${#:2}" "$1" """
]
)
target_srcs = ctx.actions.declare_file(ctx.outputs.target_srcs.basename)
ctx.actions.run_shell(
inputs = ctx.files.srcs,
outputs = [target_srcs],
command = cpSrcsCmd,
arguments = [target_srcs.path] + [src.path for src in ctx.files.srcs],
)
return target
def _purescript_tar(ctx):
target = _purescript_compile(ctx)
tar = ctx.actions.declare_file(ctx.outputs.tar.basename)
ctx.actions.run_shell(
inputs = [target],
outputs = [tar],
command = """
set -o errexit
tar --create --file "$1" --directory "$2" .
""",
arguments = [tar.path, target.path],
)
def _purescript_app(ctx):
target = _purescript_compile(ctx)
entry_params = ",".join([
'\\"{entry}\\"'.format(entry=e) for e in ctx.attr.entry_parameters
])
script = ctx.actions.declare_file(ctx.label.name)
script_content = run_template.format(
target_path = target.short_path,
entry_module = getattr(ctx.attr, "entry_module"),
entry_function = getattr(ctx.attr, "entry_function"),
entry_params = entry_params,
)
ctx.actions.write(script, script_content, is_executable = True)
runfiles = ctx.runfiles(files = [target])
return [DefaultInfo(executable = script, runfiles = runfiles)]
purescript_app = rule(
implementation = _purescript_app,
attrs = {
"srcs": attr.label_list(
allow_files = True,
),
"deps": attr.label_list(
default = [],
),
"purs": attr.label(
allow_single_file = True,
executable = True,
cfg = "host",
default = "#purs",
),
"compiler_flags": attr.string_list(
default = []
),
"entry_module": attr.string(
default = "Main",
),
"entry_function": attr.string(
default = "main",
),
"entry_parameters": attr.string_list(
default = [],
),
},
outputs = {
"target": "target",
"target_srcs": "target_srcs",
},
executable = True,
)
Called via:
purescript_app(
name = "ui",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.purs"]),
)
However, with this new version I get this error:
DEBUG: Repository io_bazel_rules_purescript instantiated at:
.../WORKSPACE:66:13: in <toplevel>
Repository rule http_archive defined at:
~/.cache/bazel/_bazel_black/b6b6a2b95964967dc5a9c8dec827e874/external/bazel_tools/tools/build_defs/repo/http.bzl:336:31: in <toplevel>
INFO: Analyzed target //webui:ui (0 packages loaded, 0 targets configured).
INFO: Found 1 target...
ERROR: .../webui/BUILD.bazel:16:15: Action webui/target failed: (Exit 127): bash failed: error executing command /nix/store/4nmqxajzaf60yjribkgvj5j54x9yvr1r-bash-5.1-p12/bin/bash -c 'set -o errexit
mkdir "$2"
"$1" compile --output "$2" "${#:3}" ' '' external/purs/purs bazel-out/k8-fastbuild/bin/webui/target ... (remaining 98 argument(s) skipped)
Use --sandbox_debug to see verbose messages from the sandbox
: line 3: external/purs/purs: No such file or directory
Target //webui:ui failed to build
Which is surprising since the file exist in the directory (external/purs/purs).
Is their anything to add in the new version?

Related

Bazel convert tar to zip

Is there a way to convert a tar target (created using rules_pkg) and turn it into a zip target in Bazel?
pkg_tar(
name = "bundle-tar",
srcs = [
":aws-lambda",
],
include_runfiles = True,
package_dir = "/",
)
pkg_zip(
name = "bundle-zip",
srcs = [
":bundle-tar", # ???
],
)
I imagine this could be accomplished using #bazel_tools//tools/zip:zipper?
This kind of works, but it is not deterministic:
genrule(
name = "bundle-zip",
srcs = [ ':bundle-tar' ],
outs = [ "bundle.zip" ],
cmd = " && ".join([
"tar xf $(location :bundle-tar)",
"zip $# $$(tar tf $(location :bundle-tar))",
]),
)
load("#bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "rules_pkg",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
"https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
],
sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
)
load("#rules_pkg//:deps.bzl", "rules_pkg_dependencies")
rules_pkg_dependencies()
Please refer to this issue for a detailed explanation on how to convert tar to zip files. Thank you.

What is the most efficient way to extract/collect files from a list of targets/providers in Bazel?

I'm writing some rules and learning Starlark as I progress.
Assume I have my own provider:
ModularResources = provider(
doc = "Modular resources",
fields = {
"artifactId": "Former Maven artifact id (don't ask me why)",
"srcs": "List of labels (a glob(..) thing)",
},
)
def _modular_resources_impl(ctx):
return ModularResources(
artifactId = ctx.attr.artifactId,
srcs = ctx.attr.srcs,
)
modular_resources = rule(
implementation = _modular_resources_impl,
attrs = {
"artifactId": attr.string(
mandatory = True,
),
"srcs": attr.label_list(
allow_files = True,
mandatory = True,
),
},
)
Then I have a generator rule which requires these:
some_generator = rule(
attrs = {
"deps": attr.label_list(
providers = [ ModularResources ]
),
...
},
...
)
In my implementation I discovered that I need to do a couple of unwraps to get the files:
def _get_files(deps):
result = []
for dep in deps:
for target in dep[ModularResources].srcs:
result += target.files.to_list()
return result
Is there a more efficient way to perform the collection?
As to why I'm doing this, the generator actually needs a special list of files like this:
def _format_files(deps):
formatted = ""
for dep in deps:
for target in dep[ModularResources].srcs:
formatted += ",".join([dep[ModularResources].artifactId + ":" + f.path for f in target.files.to_list()])
return formatted
FWIW, here is an example how this is used:
a/BUILD:
modular_resources(
name = "generator_resources",
srcs = glob(
["some/path/**/*.whatever"],
),
artifactId = "a",
visibility = ["//visibility:public"],
)
b/BUILD:
some_generator(
name = "...",
deps = [
"//a:generator_resources"
]
)
If you want to trade memory for better performance, maybe the operation can more easily be parallelised by blaze if it's done in the provider instead:
def _modular_resources_impl(ctx):
return ModularResources(
artifactId = ctx.attr.artifactId,
formatted_srcs = ",".join([artifactId + ":" + f.path for f in ctx.files.src])
)

How can you run the executables of other rules within a Bazel rule?

Say I have a custom rule, my_object. It looks like:
my_object(
name = "foo",
deps = [
//services/image-A:push,
//services/image-B:push,
]
)
Where the labels in deps are rules_docker's container_push rules.
I want to be able to bazel run //:foo and have it push the Docker images within the deps list. How do I do this?
This seems to be a specific case of just generally wanting to run the executables of other rules within the executable of a custom rule.
The thing to do here is to have my_object output an executable that executes the other executables.
Consider this example:
def _impl1(ctx):
ctx.actions.write(
output = ctx.outputs.executable,
is_executable = True,
content = "echo %s 123" % ctx.label.name)
return DefaultInfo(executable = ctx.outputs.executable)
exec_rule1 = rule(
implementation = _impl1,
executable = True,
)
def _impl2(ctx):
executable_paths = []
runfiles = ctx.runfiles()
for dep in ctx.attr.deps:
# the "./" is needed if the executable is in the current directory
# (i.e. in the workspace root)
executable_paths.append("./" + dep.files_to_run.executable.short_path)
# collect the runfiles of the other executables so their own runfiles
# will be available when the top-level executable runs
runfiles = runfiles.merge(dep.default_runfiles)
ctx.actions.write(
output = ctx.outputs.executable,
is_executable = True,
content = "\n".join(executable_paths))
return DefaultInfo(
executable = ctx.outputs.executable,
runfiles = runfiles)
exec_rule2 = rule(
implementation = _impl2,
executable = True,
attrs = {
"deps": attr.label_list(),
},
)
BUILD.bazel:
load(":defs.bzl", "exec_rule1", "exec_rule2")
exec_rule1(name = "foo")
exec_rule1(name = "bar")
exec_rule2(name = "baz", deps = [":foo", ":bar"])
and then running it:
$ bazel run //:baz
INFO: Analyzed target //:baz (4 packages loaded, 19 targets configured).
INFO: Found 1 target...
Target //:baz up-to-date:
bazel-bin/baz
INFO: Elapsed time: 0.211s, Critical Path: 0.01s
INFO: 0 processes.
INFO: Build completed successfully, 6 total actions
INFO: Build completed successfully, 6 total actions
foo 123
bar 123
I managed to achieve this by implementing DefaultInfo in rule.
def build_all_impl(ctx):
targets = ctx.attr.targets
run_files = []
for target in targets:
run_files = run_files + target.files.to_list()
DefaultInfo(
runfiles = ctx.runfiles(run_files),
)
build_all = rule(
implementation = build_all_impl,
attrs = {
"targets": attr.label_list(
doc = "target to build",
),
},
)
And then by running build_all rule
build_all(
name = "all",
targets = [
":target-1",
":target-2",
...
],
)

Bazel- How to get all transitive sources of a target

I have a need to write a rule which zips up all transitive config files (*.foo) of an executable (can be custom rule, java_binary and docker container_image).
The config files can appear on the srcs attribute of any attribute of the executable (tars, deps, runtime_deps, etc)
This sounds like it should be rather easy to do with an aspect attached to my rule but I lost my way between the various examples.
sources_aspect.bzl:
SourceFiles = provider(
fields = {
"transitive_source_files": "list of transitive source files of a target",
},
)
#add 'resources' ? if so _accumulate_transitive_config_files needs to check for dep in deps if ConfigFiles in dep
SourceAttr = ["tars", "deps", "runtime_deps", "exports"]
def _accumulate_transitive_source_files(accumulated, deps):
return depset(
transitive = [dep[SourceFiles].transitive_source_files for dep in deps] + [accumulated],
)
def _collect_current_source_files(srcs):
return [file for src in srcs for file in src.files.to_list()]
def _collect_source_files_aspect_impl(target, ctx):
current_source_files = []
if hasattr(ctx.rule.attr, "srcs"):
current_source_files = _collect_current_source_files(ctx.rule.attr.srcs)
if hasattr(ctx.rule.attr, "resources"):
current_source_files = current_source_files + _collect_current_source_files(ctx.rule.attr.resources)
accumulated_source_files = depset(current_source_files)
for attr in SourceAttr:
if hasattr(ctx.rule.attr, attr):
accumulated_source_files = _accumulate_transitive_source_files(accumulated_source_files, getattr(ctx.rule.attr, attr))
return [SourceFiles(transitive_source_files = accumulated_source_files)]
collect_source_files_aspect = aspect(
implementation = _collect_source_files_aspect_impl,
attr_aspects = SourceAttr,
)
sources.bzl
load("//sources/src/main:sources_aspect.bzl", "SourceFiles", "collect_source_files_aspect")
def _owner_to_bazel_file(fileLabel):
workspace = fileLabel.workspace_root
package = fileLabel.package
if 0 < len(workspace):
workspace = workspace + "/"
if 0 < len(package):
package = package + "/"
return workspace + package + "BUILD.bazel"
def _collect_source_files_rule_impl(ctx):
metadata = [ctx.attr.group_id, ctx.attr.artifact_id]
paths = sorted([f.path for f in ctx.attr.main_artifact_name[SourceFiles].transitive_source_files.to_list()])
owners = sorted(depset([_owner_to_bazel_file(f.owner) for f in ctx.attr.main_artifact_name[SourceFiles].transitive_source_files.to_list()] + [_owner_to_bazel_file(ctx.label)]).to_list())
ctx.actions.write(ctx.outputs.sources, "\n".join(metadata + paths + owners))
ctx.actions.write(ctx.outputs.source_files, "{\"groupId\": \"%s\", \"artifactId\": \"%s\", \"sources\": %s, \"buildFiles\": %s}" % (ctx.attr.group_id, ctx.attr.artifact_id, paths, owners))
return DefaultInfo(
runfiles = ctx.runfiles(files = [ctx.outputs.sources, ctx.outputs.source_files]),
)
source_files = rule(
implementation = _collect_source_files_rule_impl,
attrs = {
"main_artifact_name": attr.label(aspects = [collect_source_files_aspect]),
"group_id": attr.string(mandatory = True),
"artifact_id": attr.string(mandatory = True),
},
outputs = {"sources": "%{name}.sources.txt", "source_files": "%{name}.sources.json"},
)

Bazel: testng macro

I'm trying to create a macro in Bazel to wrap java_test to run testng, however I'm running into trouble passing TestNG the filename
So far I have
load("#bazel_skylib//:lib.bzl", "paths")
def java_testng(file, deps=[], **kwargs):
native.java_test(
name = paths.split_extension(file)[0],
srcs = [file],
use_testrunner=False,
main_class='org.testng.TestNG',
deps = [
"//third_party:org_testng_testng"
] + deps,
args=[file],
**kwargs
)
However args seems to be a non-existent runfile.
Help appreciated on the correct value for args
Here is a sample usage I would like
java_testng(
file = "SomeFakeTest.java",
deps = [
"//:resources",
"//third_party:com_fasterxml_jackson_core_jackson_databind",
"//third_party:org_assertj_assertj_core",
],
)
Here is the solution I came up with
load("#bazel_skylib//:lib.bzl", "paths")
def java_testng(file, deps=[], size="small", **kwargs):
native.java_library(
name = paths.split_extension(file)[0] + "-lib",
deps = [
"//third_party:org_testng_testng"
] + deps,
srcs = [file]
)
native.java_test(
name = paths.split_extension(file)[0],
use_testrunner=False,
main_class='org.testng.TestNG',
runtime_deps = [
"//third_party:org_testng_testng",
paths.split_extension(file)[0] + "-lib"
],
data = [file],
size = size,
args=["-testclass $(location " + file + ")"],
**kwargs
)
I dont know why you used a macro, I manage to call testng without.
See my solution below:
I create my program jar (using some Annotation Processor)
I create my test jar (using some Annotation Processor)
I call testng via java_test().
The alone thing I didn't figure out: how to not hardcode the "libmy-model-test-lib.jar"
java_library(
name = "my-model",
srcs = glob(["src/main/java/**/*.java"]),
resources = glob(["src/main/resources/**"]),
deps = [
"#commons_logging_jar//jar",
":lombok",
":mysema_query",
...
],
)
java_library(
name = "my-model-test-lib",
srcs = glob(["src/test/java/**/*.java"]),
deps = [
"#org_hamcrest_core_jar//jar",
"#commons_logging_jar//jar",
":lombok",
":mysema_query",
...
"#assertj_jar//jar",
"#mockito_jar//jar",
"#testng_jar//jar",
],
)
java_test(
name = "AllTests",
size = "small",
runtime_deps = [
":my-model-test-lib",
":my-model",
"#org_jboss_logging_jar//jar",
"#org_objenesis_jar//jar",
"#com_beust_jcommander//jar",
],
use_testrunner=False,
main_class='org.testng.TestNG',
args=['-testjar','libmy-model-test-lib.jar','-verbose','2'],
)
java_plugin(
name = "lombok_plugin",
processor_class = "lombok.launch.AnnotationProcessorHider$AnnotationProcessor",
deps = ["#lombok_jar//jar"],
)
java_library(
name = "lombok",
exports = ["#lombok_jar//jar"],
exported_plugins = [":lombok_plugin"],
)
java_plugin(
name = "mysema_query_plugin",
processor_class = "com.mysema.query.apt.jpa.JPAAnnotationProcessor",
deps = [
"#querydsl_apt_jar//jar",
"#mysema_codegen_jar//jar",
"#javax_persistence_jar//jar",
"#querydsl_codegen_jar//jar",
"#guava_jar//jar",
"#querydsl_core_jar//jar",
"#javax_inject_jar//jar",
],
)
java_library(
name = "mysema_query",
exports = ["#querydsl_apt_jar//jar"],
exported_plugins = [":mysema_query_plugin"],
)
java_plugin(
name = "mockito_plugin",
processor_class = "",
deps = ["#mockito_jar//jar"],
)
java_library(
name = "mockito",
exports = ["#mockito_jar//jar"],
exported_plugins = [":mockito_plugin"],
)

Resources