I would like to build many outputs based on the same input, e.g. a hex and a binary from an elf.
I will do this multiple times, different places in the wscript so I'd like to wrap it in a feature.
Ideally something like:
bld(features="hex", source="output.elf")
bld(features="bin", source="output.elf")
How would I go about implementing this?
If your elf files always have the same extension, you can simply use that:
# untested, naive code
from waflib import TaskGen
#TaskGen.extension('.elf')
def process_elf(self, node): # <- self = task gen, node is the current input node
if "bin" in self.features:
bin_node = node.change_ext('.bin')
self.create_task('make_bin_task', node, bin_node)
if "hex" in self.features:
hex_node = node.change_ext('.hex')
self.create_task('make_hex_task', node, hex_node)
If not, you have to define the features you want like that:
from waflib import TaskGen
#Taskgen.feature("hex", "bin") # <- attach method features hex AND bin
#TaskGen.before('process_source')
def transform_source(self): # <- here self = task generator
self.inputs = self.to_nodes(getattr(self, 'source', []))
self.meths.remove('process_source') # <- to disable the standard process_source
#Taskgen.feature("hex") # <- attach method to feature hex
#TaskGen.after('transform_source')
def process_hex(self):
for i in self.inputs:
self.create_task("make_hex_task", i, i.change_ext(".hex"))
#Taskgen.feature("bin") # <- attach method to feature bin
#TaskGen.after('transform_source')
def process_hex(self):
for i in self.inputs:
self.create_task("make_bin_task", i, i.change_ext(".bin"))
You have to write the two tasks make_elf_task and make_bin_task. You should put all this in a separate python file and make a "plugin".
You can also define a "shortcut" to call:
def build(bld):
bld.make_bin(source = "output.elf")
bld.make_hex(source = "output.elf")
bld(features = "hex bin", source = "output.elf") # when both needed in the same place
Like that:
from waflib.Configure import conf
#conf
def make_bin(self, *k, **kw): # <- here self = build context
kw["features"] = "bin" # <- you can add bin to existing features kw
return self(*k, **kw)
#conf
def make_hex(self, *k, **kw):
kw["features"] = "hex"
return self(*k, **kw)
Related
I'm working on a problem in which I only want to create a particular rule if a certain Bazel config has been specified (via '--config'). We have been using Bazel since 0.11 and have a bunch of build infrastructure that works around former limitations in Bazel. I am incrementally porting us up to newer versions. One of the features that was missing was compiler transitions, and so we rolled our own using configs and some external scripts.
My first attempt at solving my problem looks like this:
load("#rules_cc//cc:defs.bzl", "cc_library")
# use this with a select to pick targets to include/exclude based on config
# see __build_if_role for an example
def noop_impl(ctx):
pass
noop = rule(
implementation = noop_impl,
attrs = {
"deps": attr.label_list(),
},
)
def __sanitize(config):
if len(config) > 2 and config[:2] == "//":
config = config[2:]
return config.replace(":", "_").replace("/", "_")
def build_if_config(**kwargs):
config = kwargs['config']
kwargs.pop('config')
name = kwargs['name'] + '_' + __sanitize(config)
binary_target_name = kwargs['name']
kwargs['name'] = binary_target_name
cc_library(**kwargs)
noop(
name = name,
deps = select({
config: [ binary_target_name ],
"//conditions:default": [],
})
)
This almost gets me there, but the problem is that if I want to build a library as an output, then it becomes an intermediate dependency, and therefore gets deleted or never built.
For example, if I do this:
build_if_config(
name="some_lib",
srcs=[ "foo.c" ],
config="//:my_config",
)
and then I run
bazel build --config my_config //:some_lib
Then libsome_lib.a does not make it to bazel-out, although if I define it using cc_library, then it does.
Is there a way that I can just create the appropriate rule directly in the macro instead of creating a noop rule and using a select? Or another mechanism?
Thanks in advance for your help!
As I noted in my comment, I was misunderstanding how Bazel figures out its dependencies. The create a file section of The Rules Tutorial explains some of the details, and I followed along here for some of my solution.
Basically, the problem was not that the built files were not sticking around, it was that they were never getting built. Bazel did not know to look in the deps variable and build those things: it seems I had to create an action which uses the deps, and then register an action by returning a (list of) DefaultInfo
Below is my new noop_impl function
def noop_impl(ctx):
if len(ctx.attr.deps) == 0:
return None
# ctx.attr has the attributes of this rule
dep = ctx.attr.deps[0]
# DefaultInfo is apparently some sort of globally available
# class that can be used to index Target objects
infile = dep[DefaultInfo].files.to_list()[0]
outfile = ctx.actions.declare_file('lib' + ctx.label.name + '.a')
ctx.actions.run_shell(
inputs = [infile],
outputs = [outfile],
command = "cp %s %s" % (infile.path, outfile.path),
)
# we can also instantiate a DefaultInfo to indicate what output
# we provide
return [DefaultInfo(files = depset([outfile]))]
I have a implement a custom C compiler tool, but at the last step (linking) I am struggling to get it working. The linker produces to output files, one is the binary, and the second one is some file with additional information.
Normally you would have a wscript with something like this:
def configure(cnf):
cnf.load('my_compiler_c')
def build(bld):
bld(features='c cprogram', source='main.c', target='app.bbin')
And I could fake a second target like this
class cprogram(link_task):
run_str = (
"${LINK_CC} ${CFLAGS} ${OTHERFLAGS} "
"${INFO_FILE}${TGT[0].relpath()+'.abc'} " # TGT[0] + some string concatenating will be the app.bbin.abc file
"${CCLNK_TGT_F}${TGT[0].relpath()} " # TGT[0] this is the app.bbin file
"${CCLNK_SRC_F}${SRC} ${STLIB_MARKER} ${STLIBPATH_ST:STLIBPATH} "
"${CSTLIB_ST:CSTLIB} ${STLIB_ST:STLIB} ${LIBPATH_ST:LIBPATH} ${LIB_ST:LIB} ${LDFLAGS}"
)
ext_out = [".bbin"]
vars = ["LINKDEPS"]
But of course, with this hacky implementation waf does not know about the second target and rebuilds will not be triggered when app.bbin.abc is missing.
So how do I correctly pass two or more targets to the cprogram class?
Well, you just have to tell waf that you need two targets:
def configure(cnf):
cnf.load('my_compiler_c')
def build(bld):
bld(features='c cprogram', source='main.c', target=['app.bbin', 'app.bbin.abc'])
As I suppose you dont want to type two targets, you can use an alias to build your task generator:
# Naive, non-tested code.
from waflib.Configure import conf
#conf
def myprogram(bld, *k, **kw):
kw['features'] = "c cprogram"
add_my_abc_target_to_target(kw) # I'm lazy
return bld(*k, **kw)
You call:
def build(bld):
bld.myprogram(source='main.c', target='app.bbin')
Note: You can put all your code in a plugin, to have clean wscripts:
def configure(cnf):
cnf.load('myprogram') # loads my_c_compiler and myprogram alias
def build(bld):
bld.myprogram(source='main.c', target='app.bbin')
Situation
I have two Skylark extension rules: blah_library and blah_binary. All of a blah_library's transitive dependencies are propagated by returning a provider(transitive_deps=...), and are handled appropriately by any ultimate dependent blah_binary target.
What I want to do
I want each blah_library to also create a filegroup with all the transitive dependencies mentioned above, so that I can access them separately. E.g., I'd like to be able to pass them in as data dependencies to a cc_binary. In other words:
# Somehow have this automatically create a target named `foo__trans_deps`?
blah_library(
name = "foo",
srcs = [...],
deps = [...],
)
cc_binary(
...,
data = [":foo__trans_deps"],
)
How should I do this? Any help would be appreciated!
What I've tried
Make a macro
I tried making a macro like so:
_real_blah_library = rule(...)
def blah_library(name, *args, **kwargs):
native.filegroup(
name = name + "__trans_deps",
srcs = ???,
)
_real_blah_library(name=name, *args, **kwargs)
But I'm not sure how to access the provider provided by _real_blah_library from within the macro, so I don't know how to populate the filegroup's srcs field...
Modify the blah_library rule's implementation
Right now I have something like:
_blah_provider = provider(fields=['transitive_deps'])
def _blah_library_impl(ctx):
...
trans_deps = []
for dep in ctx.attr.deps:
trans_deps += dep[_blah_provider].trans_deps
return _blah_provider(trans_deps=trans_deps)
blah_library = rule(impl=_blah_library_impl, ...)
I tried adding the following to _blah_library_impl, but it didn't work because apparently native.filegroup can't be called within a rule's implementation ("filegroup() cannot be called during the analysis phase"):
def _blah_library_impl(ctx):
...
trans_deps = []
for dep in ctx.attr.deps:
trans_deps += dep[_blah_provider].trans_deps
native.filegroup(
name = ctx.attr.name + "__trans_deps",
srcs = trans_deps,
)
return _blah_provider(trans_deps=trans_deps)
You can't easily create a filegroup like that, but you can still achieve what you want.
If you want to use the rule in genrule.srcs, filegroup.srcs, cc_binary.data, etc., then return a DefaultInfo provider (along with _blah_provider) and set the files field to the transitive closure of files.
You can refine the solution if you want a different set of files when the rule is in a data attribute vs. when in any other (e.g. srcs): just also set the runfiles-related members in DefaultInfo. (Frankly I don't know the difference between them, I'd just set all runfiles-fields to the same value.)
I ended up making my own special filegroup-like rule, as discussed in the comments under #Laszlo's answer. Here's the raw code in case it's a useful starting point for anyone:
def _whl_deps_filegroup_impl(ctx):
input_wheels = ctx.attr.src[_PyZProvider].transitive_wheels
output_wheels = []
for wheel in input_wheels:
file_name = wheel.basename
output_wheel = ctx.actions.declare_file(file_name)
# TODO(josh): Use symlinks instead of copying. Couldn't figure out how
# to do this due to issues with constructing absolute paths...
ctx.actions.run(
outputs=[output_wheel],
inputs=[wheel],
arguments=[wheel.path, output_wheel.path],
executable="cp",
mnemonic="CopyWheel")
output_wheels.append(output_wheel)
return [DefaultInfo(files=depset(output_wheels))]
whl_deps_filegroup = rule(
_whl_deps_filegroup_impl,
attrs = {
"src": attr.label(),
},
)
I am working on an ant script to build java prjects developed with IBM RAD 7.5.
The an script is calling IBM RAD ant extenstion API. I am using Task to load the project set file(*.psf) into the memory, and calling Task to compile the projects listed in the projectSetImport.
The problem is the projects listed in psf file is not ordered by project dependency, when compiles, it fails because the depency is incorrect.
Is there any API or method to manage the dependency automatically? the psf files Iam handling is quite big, with 200+ projects in each file and it is constanly changing(e.g. some projects get removed and some new projects added in each week)
here is a detailed description for the question:
The project dependency is like:
1) project A depends on B and D.
2) project B depends on C
3) project E depends on F
A -> B -> C
A -> D
E-> F
The sample.psf file just list all projects:
A
B
C
D
E
F
loads sample.psf, which have a project list [A,B,C,D,E,F]
build project list from
the build fail at A, because A need B and D to be build first.
My current solution is to rebuild the sample.psf manually, e.g.
sample.psf file:
C
B
D
A
F
E
but this is hard to maintain, because there are 200+ projects in a psf file and they are constanly changing.
One way to attack this issue is to write a parser to read the .project file for each project, the dependency projects are listed in "projects" tag. Then implement a Directed acyclic path algorithm to reorder the dependency. This approach might be over kill. This must be a common issue in teams build IBM java projects, is there a solution?
Finally, I wrote some python code to compute the dependency. I Listed the logic below:
read the psf file into an list, the psf file is a xml file, and
the project name is in tag.
for each project in the
list, go to project source code and read the .project file and
.classpath file, these two files contains the dependency project.
for .project file(xml), fetch the project name from tag,
for .classpath file. fetch the line with attribute kind='src'
now you got [source]->[dependened_project_list], implement a
Directed acyclic map.(see attached code)
load the [source]->[dependened_project] in to the AdjecentListDigraph, and
call topoSort() to return the dependency.
generate a new ordered psf file.
/////////////////////// dap_graph.py/////////////////////////////
# -*- coding: utf-8 -*-
'''Use directed acyclic path to calculate the dependency'''
class Vertex:
def init(self, name):
self._name = name
self.visited = True
class InValidDigraphError(RuntimeError):
def init(self, arg):
self.args = arg
class AdjecentListDigraph:
'''represent a directed graph by adjacent list'''
def __init__(self):
'''use a table to store edges,
the key is the vertex name, value is vertex list
'''
self._edge_table = {}
self._vertex_name_set = set()
def __addVertex(self, vertex_name):
self._vertex_name_set.add(vertex_name)
def addEdge(self, start_vertex, end_vertex):
if not self._edge_table.has_key(start_vertex._name):
self._edge_table[start_vertex._name] = []
self._edge_table[start_vertex._name].append(end_vertex)
# populate vertex set
self.__addVertex(start_vertex._name)
self.__addVertex(end_vertex._name)
def getNextLeaf(self, vertex_name_set, edge_table):
'''pick up a vertex which has no end vertex. return vertex.name.
algorithm:
for v in vertex_set:
get vertexes not in edge_table.keys()
then get vertex whose end_vertex is empty
'''
print 'TODO: validate this is a connected tree'
leaf_set = vertex_name_set - set(edge_table.keys())
if len(leaf_set) == 0:
if len(edge_table) > 0:
raise InValidDigraphError("Error: Cyclic directed graph")
else:
vertex_name = leaf_set.pop()
vertex_name_set.remove(vertex_name)
# remove any occurrence of vertext_name in edge_table
for key, vertex_list in edge_table.items():
if vertex_name in vertex_list:
vertex_list.remove(vertex_name)
# remove the vertex who has no end vertex from edge_table
if len(vertex_list) == 0:
del edge_table[key]
return vertex_name
def topoSort(self):
'''topological sort, return list of vertex. Throw error if it is
a cyclic graph'''
sorted_vertex = []
edge_table = self.dumpEdges()
vertex_name_set = set(self.dumpVertexes())
while len(vertex_name_set) > 0:
next_vertex = self.getNextLeaf(vertex_name_set, edge_table)
sorted_vertex.append(next_vertex)
return sorted_vertex
def dumpEdges(self):
'''return the _edge_list for debugging'''
edge_table = {}
for key in self._edge_table:
if not edge_table.has_key(key):
edge_table[key] = []
edge_table[key] = [v._name for v in self._edge_table[key]]
return edge_table
def dumpVertexes(self):
return self._vertex_name_set
//////////////////////projects_loader.py///////////////////////
-- coding: utf-8 --
'''
This module will load dependencies from every projects from psf, and compute
the directed acyclic path.
Dependencies are loaded into a map structured as below:
dependency_map{"project_A":set(A1,A2,A3),
"A1:set(B1,B2,B3)}
The algorithm is:
1) read
2) call readProjectDependency(project_name)
'''
import os, xml.dom.minidom
from utils.setting import configuration
class ProjectsLoader:
def __init__(self, application_name):
self.dependency_map = {}
self.source_dir = configuration.get('Build', 'base.dir')
self.application_name = application_name
self.src_filter_list = configuration.getCollection('psf',\
'src.filter.list')
def loadDependenciesFromProjects(self, project_list):
for project_name in project_list:
self.readProjectDependency(project_name)
def readProjectDependency(self, project_name):
project_path = self.source_dir + '\\' + self.application_name + '\\'\
+ project_name
project_file_path = os.path.join(project_path,'.project')
projects_from_project_file = self.readProjectFile(project_file_path)
classpath_file_path = os.path.join(project_path,'.classpath')
projects_from_classpath_file = self.\
readClasspathFile(classpath_file_path)
projects = (projects_from_project_file | projects_from_classpath_file)
if self.dependency_map.has_key(project_name):
self.dependency_map[project_name] |= projects
else:
self.dependency_map[project_name] = projects
def loadDependencyByProjectName(self, project_name):
project_path = self.source_dir + '\\' + self.application_name + '\\'\
+ project_name
project_file_path = os.path.join(project_path,'.project')
projects_from_project_file = self.readProjectFile(project_file_path)
classpath_file_path = os.path.join(project_path,'.classpath')
projects_from_classpath_file = self.\
readClasspathFile(classpath_file_path)
projects = list(set(projects_from_project_file\
+ projects_from_classpath_file))
self.dependency_map[project_name] = projects
for project in projects:
self.loadDependencyByProjectName(project)
def readProjectFile(self, project_file_path):
DOMTree = xml.dom.minidom.parse(project_file_path)
projects = DOMTree.documentElement.getElementsByTagName('project')
return set([project.childNodes[0].data for project in projects])
def readClasspathFile(self, classpath_file_path):
dependency_projects = set([])
if os.path.isfile(classpath_file_path):
DOMTree = xml.dom.minidom.parse(classpath_file_path)
projects = DOMTree.documentElement.\
getElementsByTagName('classpathentry')
for project in projects:
if project.hasAttribute('kind') and project.getAttribute\
('kind') == 'src' and project.hasAttribute('path') and \
project.getAttribute('path') not in self.src_filter_list:
project_name = project.getAttribute('path').lstrip('/')
dependency_projects.add(project_name)
return dependency_projects
def getDependencyMap(self):
return self.dependency_map
I use the following code to temporarily modify environment variables.
#contextmanager
def _setenv(**mapping):
"""``with`` context to temporarily modify the environment variables"""
backup_values = {}
backup_remove = set()
for key, value in mapping.items():
if key in os.environ:
backup_values[key] = os.environ[key]
else:
backup_remove.add(key)
os.environ[key] = value
try:
yield
finally:
# restore old environment
for k, v in backup_values.items():
os.environ[k] = v
for k in backup_remove:
del os.environ[k]
This with context is mainly used in test cases. For example,
def test_myapp_respects_this_envvar():
with _setenv(MYAPP_PLUGINS_DIR='testsandbox/plugins'):
myapp.plugins.register()
[...]
My question: is there a simple/elegant way to write _setenv? I thought about actually doing backup = os.environ.copy() and then os.environ = backup .. but I am not sure if that would affect the program behavior (eg: if os.environ is referenced elsewhere in the Python interpreter).
I suggest you the following implementation:
import contextlib
import os
#contextlib.contextmanager
def set_env(**environ):
"""
Temporarily set the process environment variables.
>>> with set_env(PLUGINS_DIR=u'test/plugins'):
... "PLUGINS_DIR" in os.environ
True
>>> "PLUGINS_DIR" in os.environ
False
:type environ: dict[str, unicode]
:param environ: Environment variables to set
"""
old_environ = dict(os.environ)
os.environ.update(environ)
try:
yield
finally:
os.environ.clear()
os.environ.update(old_environ)
EDIT: more advanced implementation
The context manager below can be used to add/remove/update your environment variables:
import contextlib
import os
#contextlib.contextmanager
def modified_environ(*remove, **update):
"""
Temporarily updates the ``os.environ`` dictionary in-place.
The ``os.environ`` dictionary is updated in-place so that the modification
is sure to work in all situations.
:param remove: Environment variables to remove.
:param update: Dictionary of environment variables and values to add/update.
"""
env = os.environ
update = update or {}
remove = remove or []
# List of environment variables being updated or removed.
stomped = (set(update.keys()) | set(remove)) & set(env.keys())
# Environment variables and values to restore on exit.
update_after = {k: env[k] for k in stomped}
# Environment variables and values to remove on exit.
remove_after = frozenset(k for k in update if k not in env)
try:
env.update(update)
[env.pop(k, None) for k in remove]
yield
finally:
env.update(update_after)
[env.pop(k) for k in remove_after]
Usage examples:
>>> with modified_environ('HOME', LD_LIBRARY_PATH='/my/path/to/lib'):
... home = os.environ.get('HOME')
... path = os.environ.get("LD_LIBRARY_PATH")
>>> home is None
True
>>> path
'/my/path/to/lib'
>>> home = os.environ.get('HOME')
>>> path = os.environ.get("LD_LIBRARY_PATH")
>>> home is None
False
>>> path is None
True
EDIT2
A demonstration of this context manager is available on GitHub.
_environ = dict(os.environ) # or os.environ.copy()
try:
...
finally:
os.environ.clear()
os.environ.update(_environ)
I was looking to do the same thing but for unit testing, here is how I have done it using the unittest.mock.patch function:
def test_function_with_different_env_variable():
with mock.patch.dict('os.environ', {'hello': 'world'}, clear=True):
self.assertEqual(os.environ.get('hello'), 'world')
self.assertEqual(len(os.environ), 1)
Basically using unittest.mock.patch.dict with clear=True, we are making os.environ as a dictionary containing solely {'hello': 'world'}.
Removing the clear=True will let the original os.environ and add/replace the specified key/value pair inside {'hello': 'world'}.
Removing {'hello': 'world'} will just create an empty dictionary, os.envrion will thus be empty within the with.
In pytest you can temporarily set an environment variable using the monkeypatch fixture. See the docs for details. I've copied a snippet here for your convenience.
import os
import pytest
from typing import Any, NewType
# Alias for the ``type`` of monkeypatch fixture.
MonkeyPatchFixture = NewType("MonkeyPatchFixture", Any)
# This is the function we will test below to demonstrate the ``monkeypatch`` fixture.
def get_lowercase_env_var(env_var_name: str) -> str:
"""
Return the value of an environment variable. Variable value is made all lowercase.
:param env_var_name:
The name of the environment variable to return.
:return:
The value of the environment variable, with all letters in lowercase.
"""
env_variable_value = os.environ[env_var_name]
lowercase_env_variable = env_variable_value.lower()
return lowercase_env_variable
def test_get_lowercase_env_var(monkeypatch: MonkeyPatchFixture) -> None:
"""
Test that the function under test indeed returns the lowercase-ified
form of ENV_VAR_UNDER_TEST.
"""
name_of_env_var_under_test = "ENV_VAR_UNDER_TEST"
env_var_value_under_test = "EnvVarValue"
expected_result = "envvarvalue"
# KeyError because``ENV_VAR_UNDER_TEST`` was looked up in the os.environ dictionary before its value was set by ``monkeypatch``.
with pytest.raises(KeyError):
assert get_lowercase_env_var(name_of_env_var_under_test) == expected_result
# Temporarily set the environment variable's value.
monkeypatch.setenv(name_of_env_var_under_test, env_var_value_under_test)
assert get_lowercase_env_var(name_of_env_var_under_test) == expected_result
def test_get_lowercase_env_var_fails(monkeypatch: MonkeyPatchFixture) -> None:
"""
This demonstrates that ENV_VAR_UNDER_TEST is reset in every test function.
"""
env_var_name_under_test = "ENV_VAR_UNDER_TEST"
expected_result = "envvarvalue"
with pytest.raises(KeyError):
assert get_lowercase_env_var(env_var_name_under_test) == expected_result
For unit testing I prefer using a decorator function with optional parameters. This way I can use the modified environment values for a whole test function. The decorator below also restores the original environment values in case the function raises an Exception:
import os
def patch_environ(new_environ=None, clear_orig=False):
if not new_environ:
new_environ = dict()
def actual_decorator(func):
from functools import wraps
#wraps(func)
def wrapper(*args, **kwargs):
original_env = dict(os.environ)
if clear_orig:
os.environ.clear()
os.environ.update(new_environ)
try:
result = func(*args, **kwargs)
except:
raise
finally: # restore even if Exception was raised
os.environ = original_env
return result
return wrapper
return actual_decorator
Usage in unit tests:
class Something:
#staticmethod
def print_home():
home = os.environ.get('HOME', 'unknown')
print("HOME = {0}".format(home))
class SomethingTest(unittest.TestCase):
#patch_environ({'HOME': '/tmp/test'})
def test_environ_based_something(self):
Something.print_home() # prints: HOME = /tmp/test
unittest.main()