Suppose I have source-files like this:
.
├── config.json
└── test.sh
I want to create an sh_test using these files.
However, when the test runs, test.sh expects config.json to be in a different place:
.
├── configs
│ └── prod.json # config.json
└── test.sh
I do not want to rearrange my source-code to satisfy test.sh and I also don't want to add a bunch of file copying/moving to the start of test.sh.
Is it possible to tell Bazel how to arrange the files for test.sh in an arbitrary way?
Ideal syntax would be like this:
sh_test(
name = "test",
srcs = [ "test.sh" ],
data = {
"configs/prod.json": "config.json",
},
)
Use a genrule:
genrule(
name = "prod_json",
srcs = ["config.json"],
outs = ["configs/prod.json"],
cmd = "cp $< $#",
)
sh_test(
name = "test",
srcs = ["test.sh"],
data = ["configs/prod.json"],
)
Alternatively, you could write a macro that matches the syntax in your question:
# BUILD
load(":sh_test.bzl", "sh_test")
sh_test(
name = "test",
srcs = ["test.sh"],
data = {
"configs/test.json": "config_test.json",
"configs/staging.json": "config_staging.json",
"configs/prod.json": "config.json",
}
)
# sh_test.bzl
def sh_test(name, data = {}, **kwargs):
datas = []
for dest, src in data.items():
genrule_name = dest.replace("/", "_")
native.genrule(
name = genrule_name,
srcs = [src],
outs = [dest],
cmd = "cp $< $#",
)
datas.append(genrule_name)
native.sh_binary(
name = name,
data = datas,
**kwargs
)
Related
I am trying to consume some Python code + C extensions that are produced by CMake.
I'm using rules_foreign_cc to build the code, and it puts the code into an output directory, but I'm stumped as to how I can turn that into a py_library. I tried the below:
Depend on the cmake rule as srcs
Set imports to try to point to the directory containing the Python packages
But it doesn't work. When I run the py_test, the constructed PYTHONPATH doesn't point to the directory containing the python package from the cmake rule.
Thanks for any help!
Here's what I've got in my third_party/BUILD.bazel:
load("#rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
load("#rules_python//python:defs.bzl", "py_library", "py_test")
filegroup(
name = "torch_mlir_src",
srcs = glob(["torch-mlir/**"]),
)
make(
name = "torch_mlir",
...
install = False,
lib_source = ":torch_mlir_src",
out_data_dirs = ["python_packages"],
postfix_script = " && ".join([
"cp -r --dereference tools/torch-mlir/python_packages/torch_mlir $$INSTALLDIR$$/python_packages",
]),
targets = ["tools/torch-mlir/all"],
working_directory = "external/llvm-project/llvm",
)
# This doesn't seem to work.
py_library(
name = "torch_mlir_py",
srcs = [":torch_mlir"],
imports = ["$(BINDIR)/third_party/torch_mlir/python_packages"],
srcs_version = "PY3",
)
py_test(
name = "torch_mlir_annotations_sugar_test",
srcs = ["torch-mlir/python/test/annotations-sugar.py"],
main = "torch-mlir/python/test/annotations-sugar.py",
srcs_version = "PY3",
deps = [":torch_mlir_py"],
)
I have a cc_binary target that uses runfiles. I would like to zip the executable and all of the runfiles into a single archive using a genrule.
Something like:
genrule(
name = "zip_binary"
srcs = [
":binary",
],
outs = [
"binary.zip",
],
cmd = "zip -r $(OUTS) $(locations :binary)",
)
However, this only includes the binary and not the binary.runfiles dir.
How can I get bazel to include the runfiles in the srcs?
Genrules don't have access to enough information to do that. With a full custom rule it's pretty easy though. Something like this:
def _zipper_impl(ctx):
inputs = ctx.runfiles(files = ctx.files.srcs)
inputs = inputs.merge_all([a[DefaultInfo].default_runfiles for a in ctx.attr.srcs])
ctx.actions.run_shell(
outputs = [ctx.output.out],
inputs = inputs.files,
command = " ".join(["zip", "-r", ctx.output.out.path] +
[i.path for i in inputs.files.to_list()]),
)
return [DefaultInfo(files = depset(ctx.output.out))]
zipper = rule(
impl = _zipper_impl,
attrs = {
"out": attr.output(mandatory = True),
"srcs": attr.label_list(allow_files = True),
},
)
I have some html files in "root/html_files/*.html" directory. I want to iterate on these html files and run some bazel rules on them, from "root/tests/" directory.
So, I made a filegroup of those html files and tried to get access to them in "root/tests/" directory, but that is't working.
I wonder if it is possible?
My BUILD file in "root/" directory:
HTMLS_LIST = glob(["html_files/*.html",])
filegroup(
name = "html_files",
srcs = HTMLS_LIST,
visibility = [ "//visibility:public" ],)
My BUILD file in "root/tests/" directory:
load("//tests:automation_test.bzl", "make_and_run_html_tests")
make_and_run_html_tests(
name = 'test_all_htmls',
srcs = ['test/automation_test.py'],
html_files = '//:html_files')
My bzl file in "root/tests/" directory:
def make_and_run_html_tests(name, html_files, srcs):
tests = []
for html_file in html_files: # I want to iterate on sources of html filegroup here
folders = html_file.split("/")
filename = folders[-1].split(".")[0]
test_rule_name = 'test_' + filename + '_file'
native.py_test(
name = test_rule_name,
srcs = srcs,
main = srcs[0],
data = [
html_file,
],
args = [html_file],
)
testname = ":" + test_rule_name
tests.append(testname)
native.test_suite(
name = name,
tests = tests,
)
And my python unittest file in "root/tests/" directory:
import sys
import codecs
import unittest
class TestHtmlDocumetns(unittest.TestCase):
def setUp(self):
self.html_file_path = sys.argv[1]
def test_html_file(self):
fid = codecs.open(self.html_file_path, 'r')
print(fid.read())
self.assertTrue(fid)
if __name__ == '__main__':
unittest.main(argv=[sys.argv[1]])
You can't access the references to the files inside another filegroup / rule from within a macro like that. You'd need to create a rule and access them via the ctx.files attr
However, you can iterate over them if you were to remove the filegroup, and pass the glob directly to the macro:
HTMLS_LIST = glob(["html_files/*.html"])
make_and_run_html_tests(
name = 'test_all_htmls',
srcs = ['test/automation_test.py'],
html_files = HTMLS_LIST
)
The glob is resolved to an array before expanding the macro
In the Bazel official documentation there is an example explaining how to create a Java library built from regular java files and files generated by a :gen_java_srcs rule. I rewrite this code here for ease of reading:
java_library(
name = "mylib",
srcs = glob(["*.java"]) + [":gen_java_srcs"],
deps = "...",
)
genrule(
name = "gen_java_srcs",
outs = [
"Foo.java",
"Bar.java",
],
...
)
Now in a C++ perspective, I am in a scenario where the genrule generates two kind of files: .hpp and .cpp:
genrule(
name = "gen_cpp_srcs",
outs = [
"myFile_1.hpp","myFile_2.hpp",...,"myFile_N.hpp",
"myFile.cpp","myFile_2.cpp",...,"myFile_N.cpp",
],
...
)
where N is some tens.
My problem/question is: how to write the cc_library rule, with an automatic dispatching of the hpp and cpp files into hdrs and srcs field?
I want something like:
cc_library(
name = "mylib",
srcs = glob(["*.cpp"]) + (howto: .cpp files of [":gen_cpp_srcs"]),
hdrs = glob(["*.hpp"]) + (howto: .hpp files of [":gen_cpp_srcs"]),
...
)
Some magic like:
output_filter(":gen_cpp_srcs","*.cpp")
would be perfect, but I do not know enough of Bazel to make it real.
Globs only get expanded when they're passed into rules, so you'll need to write a simple rule. I would package it like this (in a file named filter.bzl):
# The actual rule which does the filtering.
def _do_filter_impl(ctx):
return struct(
files = set([f for f in ctx.files.srcs if f.path.endswith(ctx.attr.suffix)]),
)
_do_filter = rule(
implementation = _do_filter_impl,
attrs = {
"srcs": attr.label_list(
mandatory = True,
allow_files = True,
),
"suffix": attr.string(
mandatory = True,
),
},
)
# A convenient macro to wrap the custom rule and cc_library.
def filtered_cc_library(name, srcs, hdrs, **kwargs):
_do_filter(
name = "%s_hdrs" % name,
visibility = ["//visibility:private"],
srcs = hdrs,
suffix = ".hpp",
)
_do_filter(
name = "%s_srcs" % name,
visibility = ["//visibility:private"],
srcs = srcs,
suffix = ".cpp",
)
native.cc_library(
name = name,
srcs = [ ":%s_srcs" % name ],
hdrs = [ ":%s_hdrs" % name ],
**kwargs
)
This is what my demo BUILD file looks like (I changed the globs so they both include *.cpp and *.hpp files; using the label of a genrule will work the same way):
load("//:filter.bzl", "filtered_cc_library")
filtered_cc_library(
name = "mylib",
srcs = glob(["*.*pp"]),
hdrs = glob(["*.*pp"]),
)
This is easy to extend to more sophisticated filtering by changing _do_filter_impl. In particular, changing suffix to an attr.string_list so you can accept multiple C/C++ source/header extensions seems like a good idea.
Depending on the genrule by name (:gen_cpp_srcs) will give you all of the outputs of the genrule, as you have noted. Instead, you can depend on the individual outputs of the genrule (e.g. hdrs = [:myFile.hpp] and srcs = [:myFile.cpp]).
See also the answer to Bazel & automatically generated cpp / hpp files.
Looks like you know the total number of files that should be generated. Can you put those in their own variables and then reuse them in both targets. Something like this in your BUILD file:
output_cpp_files = [
"myFile_1.cpp",
"myFile_2.cpp",
"myFile_3.cpp"
]
output_hpp_files = [
"myFile_1.hpp",
"myFile_2.hpp",
"myFile_3.hpp"
]
genrule(
name = "gen_cpp_srcs",
outs = output_cpp_files + output_hpp_files,
cmd = """
touch $(OUTS)
"""
)
cc_library(
name = "mylib",
srcs = output_cpp_files,
hdrs = output_hpp_files
)
What I figured currently is creating a AllTest and run it with junit. But, I am not satisfied with it. I want this rule can create as many tests as many java test file in created in codebase.
def junit_suite_test(name, srcs, deps, size="small", resources=[], classpath_resources=[], jvm_flags=[], tags=[], data=[]):
tests = []
package = PACKAGE_NAME.replace("src/test/java/", "").replace("/", ".")
for src in srcs:
if src.endswith("Test.java"):
if "/" in src:
src = package + "." + src.replace("/", ".")
tests += [src.replace(".java", ".class")]
native.genrule(
name = name + "-AllTests-gen",
outs = ["AllTests.java"],
cmd = """
cat <<EOF >> $#
package %s;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
#RunWith(Suite.class)
#Suite.SuiteClasses({%s})
public class AllTests {}
EOF
""" % (package, ",".join(tests))
)
native.java_test(
name = name,
srcs = srcs + ["AllTests.java"],
test_class = package + ".AllTests",
resources = resources,
classpath_resources = classpath_resources,
data = data,
size = size,
tags = tags,
jvm_flags = jvm_flags,
deps = deps + [
],
)
Hi you can do something like that:
[java_test(name = s[:-5], srcs = s) for s in glob(["*.java"])]
That will create on test target per java file.
With that method, your macro would looks like:
def junit_suite_test(name, srcs, deps, size="small", resources=[], classpath_resources=[], jvm_flags=[], tags=[], data=[]):
[native.java_test(
name = name,
srcs = src,
resources = resources,
classpath_resources = classpath_resources,
data = data,
size = size,
tags = tags,
jvm_flags = jvm_flags,
deps = deps,
) for src in srcs if src.endswith("Test.java")]
Of course you probably needs some adaptation to feed in the good sources.
However, I would recommend against doing that over your solution as too much parallelisms can actually be slower in fine. The test log and the XML file will report the actual failing test case and you can use shard_count to increase parallelism is really needed.