I am trying to run a module with the platforms flag:
--platforms=//:gpu_server
If it is specified, it needs to load the following chunk of code in the WORKSPACE:
pip_parse(
name = "python_perception_libs",
requirements_lock = "//third_party:python_requirements_lock_perception.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_perception_libs//:requirements.bzl", "install_deps")
install_deps()
if it is not specified then it should load
pip_parse(
name = "python_libs",
requirements_lock = "//third_party:python_requirements_lock.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_libs//:requirements.bzl", "install_deps")
install_deps()
I have no clue how to actually approach this, and various experiments with select have yielded no results: For example:
# Define GPU constraint values
constraint_setting(name = "gpu")
constraint_value(name = "turing", constraint_setting = "gpu")
constraint_value(name = "ampere", constraint_setting = "gpu")
constraint_value(name = "none", constraint_setting = "gpu")
# Platform
platform(
name = "gpu_server",
constraint_values = [
"#platforms//os:linux",
"#platforms//cpu:x86_64",
":gpu",
],
)
select({
"#platforms//os:linux": {
pip_parse(
name = "python_perception_libs",
requirements_lock = "//third_party:python_requirements_lock_perception.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_perception_libs//:requirements.bzl", "install_deps")
install_deps()
},
"//conditions:default": {
pip_parse(
name = "python_libs",
requirements_lock = "//third_party:python_requirements_lock.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_libs//:requirements.bzl", "install_deps")
install_deps()
},
})
This does not work at all
WORKSPACE and BUILD files are more declarative than imperative (bzl files are imperative). select() isn't really like an if statement, it's more declaring the possible values an attribute can take, depending on what conditions are true at the the time the target is evaluated.
Similar to https://stackoverflow.com/a/71916582/6327445, declare both (using keyword arguments to disambiguate the symbols in the load() calls), and then elsewhere (e.g. in BUILD files) use one or the other, or use them in select():
pip_parse(
name = "python_perception_libs",
requirements_lock = "//third_party:python_requirements_lock_perception.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_perception_libs//:requirements.bzl", python_perception_libs_install_deps = "install_deps")
python_perception_libs_install_deps()
pip_parse(
name = "python_libs",
requirements_lock = "//third_party:python_requirements_lock.txt",
python_interpreter_target = "#python//:bin/python3",
)
load("#python_libs//:requirements.bzl", python_libs_install_deps = "install_deps")
python_libs_install_deps()
then in a BUILD file, could be something like this:
your_rule(
...
some_attribute = select({
":some_condition": "#python_perception_libs//package:target",
":other_condition": "#python_libs//package:target",
})
...
)
Your actual usage depends on what you're trying to do overall.
Related
I am using bazel 3.7.2.
I am getting this error
error loading package '#maven//': Unable to find package for #io_bazel_stardoc//stardoc:stardoc.bzl: The repository '#io_bazel_stardoc' could not be resolved.
This is my workspace
load("#bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "bazel_skylib",
urls = [
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
"https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
],
sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
)
load("#bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
bazel_skylib_workspace()
RULES_JVM_EXTERNAL_TAG = "4.0"
RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
http_archive(
name = "maven",
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
sha256 = RULES_JVM_EXTERNAL_SHA,
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
)
load("#maven//:defs.bzl", "maven_install")
maven_install(
artifacts = [
"com.fasterxml.jackson.core:jackson-databind:2.12.1",
"org.apache.commons:commons-lang3:3.11"
],
repositories = [
"https://repo1.maven.org/maven2",
"https://jcenter.bintray.com/"
],
);
I found the solution
git_repository(
name = "io_bazel_stardoc",
remote = "https://github.com/bazelbuild/stardoc.git",
tag = "0.4.0",
)
load("#io_bazel_stardoc//:setup.bzl", "stardoc_repositories")
stardoc_repositories()
I'm writing some rules and learning Starlark as I progress.
Assume I have my own provider:
ModularResources = provider(
doc = "Modular resources",
fields = {
"artifactId": "Former Maven artifact id (don't ask me why)",
"srcs": "List of labels (a glob(..) thing)",
},
)
def _modular_resources_impl(ctx):
return ModularResources(
artifactId = ctx.attr.artifactId,
srcs = ctx.attr.srcs,
)
modular_resources = rule(
implementation = _modular_resources_impl,
attrs = {
"artifactId": attr.string(
mandatory = True,
),
"srcs": attr.label_list(
allow_files = True,
mandatory = True,
),
},
)
Then I have a generator rule which requires these:
some_generator = rule(
attrs = {
"deps": attr.label_list(
providers = [ ModularResources ]
),
...
},
...
)
In my implementation I discovered that I need to do a couple of unwraps to get the files:
def _get_files(deps):
result = []
for dep in deps:
for target in dep[ModularResources].srcs:
result += target.files.to_list()
return result
Is there a more efficient way to perform the collection?
As to why I'm doing this, the generator actually needs a special list of files like this:
def _format_files(deps):
formatted = ""
for dep in deps:
for target in dep[ModularResources].srcs:
formatted += ",".join([dep[ModularResources].artifactId + ":" + f.path for f in target.files.to_list()])
return formatted
FWIW, here is an example how this is used:
a/BUILD:
modular_resources(
name = "generator_resources",
srcs = glob(
["some/path/**/*.whatever"],
),
artifactId = "a",
visibility = ["//visibility:public"],
)
b/BUILD:
some_generator(
name = "...",
deps = [
"//a:generator_resources"
]
)
If you want to trade memory for better performance, maybe the operation can more easily be parallelised by blaze if it's done in the provider instead:
def _modular_resources_impl(ctx):
return ModularResources(
artifactId = ctx.attr.artifactId,
formatted_srcs = ",".join([artifactId + ":" + f.path for f in ctx.files.src])
)
I am running into a build error for a bazel target. I have checked the code and I could not find anything wrong. I suspect I might be looking at the wrong version of the code. Is there a way to print out tag/versions/hash of the all the code packages a target is dependent on?
bazel query 'deps(//my:target)' --nohost_deps --noimplicit_deps --output=build
This will print out the BUILD targets of all explicit dependencies a target depends on in the BUILD file format. Here's an example output of running that command in a real project:
# /home/user/code/rules_jvm_external/tests/integration/BUILD:12:1
java_test(
name = "GlobalArtifactExclusionsTest",
deps = ["#global_exclusion_testing//:com_diffplug_durian_durian_core", "#global_exclusion_testing//:com_google_guava_guava", "#global_exclusion_testing//:com_squareup_okhttp3_okhttp", "#maven//:org_hamcrest_hamcrest", "#maven//:org_hamcrest_hamcrest_core"],
srcs = ["//tests/integration:GlobalArtifactExclusionsTest.java"],
test_class = "com.jvm.external.GlobalArtifactExclusionsTest",
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/maven/BUILD:103:1
jvm_import(
name = "org_hamcrest_hamcrest_core",
tags = ["maven_coordinates=org.hamcrest:hamcrest-core:2.1"],
jars = ["#maven//:v1/https/jcenter.bintray.com/org/hamcrest/hamcrest-core/2.1/hamcrest-core-2.1.jar"],
deps = ["#maven//:org_hamcrest_hamcrest"],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/maven/BUILD:115:1
jvm_import(
name = "org_hamcrest_hamcrest",
tags = ["maven_coordinates=org.hamcrest:hamcrest:2.1"],
jars = ["#maven//:v1/https/jcenter.bintray.com/org/hamcrest/hamcrest/2.1/hamcrest-2.1.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:79:1
jvm_import(
name = "com_squareup_okhttp3_okhttp",
tags = ["maven_coordinates=com.squareup.okhttp3:okhttp:3.14.1"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/3.14.1/okhttp-3.14.1.jar"],
deps = ["#global_exclusion_testing//:com_squareup_okio_okio"],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:91:1
jvm_import(
name = "com_squareup_okio_okio",
tags = ["maven_coordinates=com.squareup.okio:okio:1.17.2"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/squareup/okio/okio/1.17.2/okio-1.17.2.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:52:1
jvm_import(
name = "com_google_guava_guava",
tags = ["maven_coordinates=com.google.guava:guava:27.0-jre"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar"],
deps = ["#global_exclusion_testing//:com_google_guava_listenablefuture", "#global_exclusion_testing//:com_google_code_findbugs_jsr305", "#global_exclusion_testing//:com_google_guava_failureaccess", "#global_exclusion_testing//:com_google_errorprone_error_prone_annotations", "#global_exclusion_testing//:org_checkerframework_checker_qual"],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:102:1
jvm_import(
name = "org_checkerframework_checker_qual",
tags = ["maven_coordinates=org.checkerframework:checker-qual:2.5.2"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:68:1
jvm_import(
name = "com_google_guava_listenablefuture",
tags = ["maven_coordinates=com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:41:1
jvm_import(
name = "com_google_guava_failureaccess",
tags = ["maven_coordinates=com.google.guava:failureaccess:1.0"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:30:1
jvm_import(
name = "com_google_errorprone_error_prone_annotations",
tags = ["maven_coordinates=com.google.errorprone:error_prone_annotations:2.2.0"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:19:1
jvm_import(
name = "com_google_code_findbugs_jsr305",
tags = ["maven_coordinates=com.google.code.findbugs:jsr305:3.0.2"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar"],
deps = [],
)
# /home/user/.cache/bazel/_bazel_user/8484bc4fff18ee4a905b69a9ddb0e143/external/global_exclusion_testing/BUILD:8:1
jvm_import(
name = "com_diffplug_durian_durian_core",
tags = ["maven_coordinates=com.diffplug.durian:durian-core:1.2.0"],
jars = ["#global_exclusion_testing//:v1/https/repo1.maven.org/maven2/com/diffplug/durian/durian-core/1.2.0/durian-core-1.2.0.jar"],
deps = [],
)
I have a need to write a rule which zips up all transitive config files (*.foo) of an executable (can be custom rule, java_binary and docker container_image).
The config files can appear on the srcs attribute of any attribute of the executable (tars, deps, runtime_deps, etc)
This sounds like it should be rather easy to do with an aspect attached to my rule but I lost my way between the various examples.
sources_aspect.bzl:
SourceFiles = provider(
fields = {
"transitive_source_files": "list of transitive source files of a target",
},
)
#add 'resources' ? if so _accumulate_transitive_config_files needs to check for dep in deps if ConfigFiles in dep
SourceAttr = ["tars", "deps", "runtime_deps", "exports"]
def _accumulate_transitive_source_files(accumulated, deps):
return depset(
transitive = [dep[SourceFiles].transitive_source_files for dep in deps] + [accumulated],
)
def _collect_current_source_files(srcs):
return [file for src in srcs for file in src.files.to_list()]
def _collect_source_files_aspect_impl(target, ctx):
current_source_files = []
if hasattr(ctx.rule.attr, "srcs"):
current_source_files = _collect_current_source_files(ctx.rule.attr.srcs)
if hasattr(ctx.rule.attr, "resources"):
current_source_files = current_source_files + _collect_current_source_files(ctx.rule.attr.resources)
accumulated_source_files = depset(current_source_files)
for attr in SourceAttr:
if hasattr(ctx.rule.attr, attr):
accumulated_source_files = _accumulate_transitive_source_files(accumulated_source_files, getattr(ctx.rule.attr, attr))
return [SourceFiles(transitive_source_files = accumulated_source_files)]
collect_source_files_aspect = aspect(
implementation = _collect_source_files_aspect_impl,
attr_aspects = SourceAttr,
)
sources.bzl
load("//sources/src/main:sources_aspect.bzl", "SourceFiles", "collect_source_files_aspect")
def _owner_to_bazel_file(fileLabel):
workspace = fileLabel.workspace_root
package = fileLabel.package
if 0 < len(workspace):
workspace = workspace + "/"
if 0 < len(package):
package = package + "/"
return workspace + package + "BUILD.bazel"
def _collect_source_files_rule_impl(ctx):
metadata = [ctx.attr.group_id, ctx.attr.artifact_id]
paths = sorted([f.path for f in ctx.attr.main_artifact_name[SourceFiles].transitive_source_files.to_list()])
owners = sorted(depset([_owner_to_bazel_file(f.owner) for f in ctx.attr.main_artifact_name[SourceFiles].transitive_source_files.to_list()] + [_owner_to_bazel_file(ctx.label)]).to_list())
ctx.actions.write(ctx.outputs.sources, "\n".join(metadata + paths + owners))
ctx.actions.write(ctx.outputs.source_files, "{\"groupId\": \"%s\", \"artifactId\": \"%s\", \"sources\": %s, \"buildFiles\": %s}" % (ctx.attr.group_id, ctx.attr.artifact_id, paths, owners))
return DefaultInfo(
runfiles = ctx.runfiles(files = [ctx.outputs.sources, ctx.outputs.source_files]),
)
source_files = rule(
implementation = _collect_source_files_rule_impl,
attrs = {
"main_artifact_name": attr.label(aspects = [collect_source_files_aspect]),
"group_id": attr.string(mandatory = True),
"artifact_id": attr.string(mandatory = True),
},
outputs = {"sources": "%{name}.sources.txt", "source_files": "%{name}.sources.json"},
)
I'm trying to create a macro in Bazel to wrap java_test to run testng, however I'm running into trouble passing TestNG the filename
So far I have
load("#bazel_skylib//:lib.bzl", "paths")
def java_testng(file, deps=[], **kwargs):
native.java_test(
name = paths.split_extension(file)[0],
srcs = [file],
use_testrunner=False,
main_class='org.testng.TestNG',
deps = [
"//third_party:org_testng_testng"
] + deps,
args=[file],
**kwargs
)
However args seems to be a non-existent runfile.
Help appreciated on the correct value for args
Here is a sample usage I would like
java_testng(
file = "SomeFakeTest.java",
deps = [
"//:resources",
"//third_party:com_fasterxml_jackson_core_jackson_databind",
"//third_party:org_assertj_assertj_core",
],
)
Here is the solution I came up with
load("#bazel_skylib//:lib.bzl", "paths")
def java_testng(file, deps=[], size="small", **kwargs):
native.java_library(
name = paths.split_extension(file)[0] + "-lib",
deps = [
"//third_party:org_testng_testng"
] + deps,
srcs = [file]
)
native.java_test(
name = paths.split_extension(file)[0],
use_testrunner=False,
main_class='org.testng.TestNG',
runtime_deps = [
"//third_party:org_testng_testng",
paths.split_extension(file)[0] + "-lib"
],
data = [file],
size = size,
args=["-testclass $(location " + file + ")"],
**kwargs
)
I dont know why you used a macro, I manage to call testng without.
See my solution below:
I create my program jar (using some Annotation Processor)
I create my test jar (using some Annotation Processor)
I call testng via java_test().
The alone thing I didn't figure out: how to not hardcode the "libmy-model-test-lib.jar"
java_library(
name = "my-model",
srcs = glob(["src/main/java/**/*.java"]),
resources = glob(["src/main/resources/**"]),
deps = [
"#commons_logging_jar//jar",
":lombok",
":mysema_query",
...
],
)
java_library(
name = "my-model-test-lib",
srcs = glob(["src/test/java/**/*.java"]),
deps = [
"#org_hamcrest_core_jar//jar",
"#commons_logging_jar//jar",
":lombok",
":mysema_query",
...
"#assertj_jar//jar",
"#mockito_jar//jar",
"#testng_jar//jar",
],
)
java_test(
name = "AllTests",
size = "small",
runtime_deps = [
":my-model-test-lib",
":my-model",
"#org_jboss_logging_jar//jar",
"#org_objenesis_jar//jar",
"#com_beust_jcommander//jar",
],
use_testrunner=False,
main_class='org.testng.TestNG',
args=['-testjar','libmy-model-test-lib.jar','-verbose','2'],
)
java_plugin(
name = "lombok_plugin",
processor_class = "lombok.launch.AnnotationProcessorHider$AnnotationProcessor",
deps = ["#lombok_jar//jar"],
)
java_library(
name = "lombok",
exports = ["#lombok_jar//jar"],
exported_plugins = [":lombok_plugin"],
)
java_plugin(
name = "mysema_query_plugin",
processor_class = "com.mysema.query.apt.jpa.JPAAnnotationProcessor",
deps = [
"#querydsl_apt_jar//jar",
"#mysema_codegen_jar//jar",
"#javax_persistence_jar//jar",
"#querydsl_codegen_jar//jar",
"#guava_jar//jar",
"#querydsl_core_jar//jar",
"#javax_inject_jar//jar",
],
)
java_library(
name = "mysema_query",
exports = ["#querydsl_apt_jar//jar"],
exported_plugins = [":mysema_query_plugin"],
)
java_plugin(
name = "mockito_plugin",
processor_class = "",
deps = ["#mockito_jar//jar"],
)
java_library(
name = "mockito",
exports = ["#mockito_jar//jar"],
exported_plugins = [":mockito_plugin"],
)