The Python Part
I have a python application with multiple entrypoints, json_out and json_in. I can run them both with this default.nix
with import <nixpkgs> {};
(
let jsonio = python37.pkgs.buildPythonPackage rec {
pname = "jsonio";
version = "0.0.1";
src = ./.;
};
in python37.withPackages (ps: [ jsonio ])
).env
Like so:
$ nix-shell --run "json_out"
{ "a" : 1, "b", 2 }
$ nix-shell --run "echo { \"a\" : 1, \"b\", 2 } | json_in"
keys: a,b
values: 1,2
The System Part
I want to also invoke jq in the nix shell, like this:
$ nix-shell --run --pure "json_out | jq '.a' | json_in"
But I can't because it is not included. I know that I can include jq into the nix shell using this default.nix
with import <nixpkgs> {};
stdenv.mkDerivation rec {
name = "jsonio-environment";
buildInputs = [ pkgs.jq ];
}
And it works on its own:
$ nix-shell --run --pure "echo { \"a\" : 1, \"b\", 2 } | jq '.a'"
{ "a" : 1 }
But now I don't have my application:
$ nix-shell --run "json_out | jq '.a'"
/tmp/nix-shell-20108-0/rc: line 1: json_out: command not found
The Question
What default.nix file can I provide that will include both my application and the jq package?
My preferred way to achieve this is to use .overrideAttrs to add additional dependencies to the environment like so:
with import <nixpkgs> {};
(
let jsonio = python37.pkgs.buildPythonPackage rec {
pname = "jsonio";
version = "0.0.1";
src = ./.;
};
in python37.withPackages (ps: [jsonio ])
).env.overrideAttrs (drv: {
buildInputs = [ jq ];
})
I needed to:
provide the output of buildPythonPackage as part of the input of mkDerivation
omit the env. Based on a hint from an error message:
Python 'env' attributes are intended for interactive nix-shell
sessions, not for building!
Here's what I ended up with:
with import <nixpkgs> {};
let jsonio_installed = (
let jsonio_module = (
python37.pkgs.buildPythonPackage rec {
pname = "jsonio";
version = "0.0.1";
src = ./.;
}
);
in python37.withPackages (ps: [jsonio_module ])
);
in stdenv.mkDerivation rec {
name = "jsonio-environment";
buildInputs = [ pkgs.jq jsonio_installed ];
}
Related
I'm using the please build tool and I'm having trouble using the target|file syntax with a filegroup. Specifically, I want to use specific files from the filegroup output in a genrule, but the syntax doesn't seem to work. Here's an example of what I'm trying to do:
# subdir/BUILD.plz
this = package_name()
a = text_file(
name = 'a_file',
content = "aaaaaaaaa",
)
b = text_file(
name = 'b_file',
content = "bbbbbbbbb",
)
c = text_file(
name = 'c_file',
content = "ccccccccc",
)
filegroup(
name = this,
srcs = {
'A': [a],
'B': [b],
'C': [c],
},
visibility = ['PUBLIC']
)
# BUILD.plz
sub = '#//subdir'
a_and_c_concat = genrule(
name = 'a_and_c_concat',
srcs = {
'A': [f'{sub}|A'],
'C': [f'{sub}|C'],
},
outs = ["out"],
cmd = """
set -eux
cat "${SRCS_A}" "${SRCS_C}" > "${OUTS}"
""",
visibility = ['PUBLIC'],
)
It does the expected behavior if I change the filegroup to be a genrule and 're-export' srcs to outs like so:
# subdir/BUILD.plz
...
genrule(
name = this,
srcs = {
'A': [a],
'B': [b],
'C': [c],
},
outs = {
'A': ['a'],
'B': ['b'],
'C': ['c'],
},
cmd = """
set -eux
cp "${SRCS_A}" "${OUTS_A}"
cp "${SRCS_B}" "${OUTS_B}"
cp "${SRCS_C}" "${OUTS_C}"
""",
visibility = ['PUBLIC']
)
Is there a way to use the target|file syntax with filegroup in Please build, or is there another way to achieve the same effect another way?
Any help or advice would be greatly appreciated. Thank you!
This isn't implemented at the moment, but I think it's a reasonable thing for Please to do.
Filegroups are just a thin wrapper around build_rule(), so they accept all the same arguments, however they don't expose named sources as named outputs. I've created an issue here to track this as a feature request:
https://github.com/thought-machine/please/issues/2701
when use the python tool to generate the .cpp/.hpp code like the protobuf tool, but I don't know how many files will be generated, so it's a little not the same as protbuf tool.
In one genrule:
def __generate_core_ifce_impl(ctx):
...
output_file = ctx.actions.declare_directory(out)
cmd = """
mkdir -p {path};
""".format(path = output_file.path)
cmd += """
{tools} -i {src} -o {output_dir}
""".format(tools = tools, src = ctx.files.srcs, output_dir = output_file.path)
ctx.actions.run_shell(
command = cmd,
inputs = ctx.files.srcs,
outputs = [output_file]
)
return [DefaultInfo(files = depset([output_file])),]
_generate_core_ifce = rule (
implementation = __generate_core_ifce_impl,
attrs = {
"srcs": attr.label_list(mandatory = False, allow_files = True),
"tools": attr.label_list(mandatory = True, allow_files = True),
"out": attr.sting(mandatory = True),
},
)
In output_file directory , there will generate some *.cpp && *.hpp, but i can't know their names
then in another rule , cc_library will use *.cpp && *.hpp which are in output_file directory
the questions is: how to write this rule?
I can't get the files in the output_file diectory,
so I can't write the cc_library?
You should be able to use the name of the target, and the cc_library will use the files that are given in the DefaultInfo, e.g.:
_generate_core_ifce(
name = "my_generate_core_ifce_target",
...
)
cc_library(
name = "my_cc_library_target",
srcs = [":my_generate_core_ifce_target"],
...
)
edit: adding an example:
BUILD:
load(":defs.bzl", "my_rule")
my_rule(
name = "my_target",
)
cc_binary(
name = "cc",
srcs = [":my_target"],
)
defs.bzl:
def _impl(ctx):
output_dir = ctx.actions.declare_directory("my_outputs")
command = """
mkdir -p {output_dir}
cat > {output_dir}/main.c <<EOF
#include "stdio.h"
#include "mylib.h"
int main() {
printf("hello world %d\\n", get_num());
return 0;
}
EOF
cat > {output_dir}/mylib.c <<EOF
int get_num() {
return 42;
}
EOF
cat > {output_dir}/mylib.h <<EOF
int get_num();
EOF
""".replace("{output_dir}", output_dir.path)
ctx.actions.run_shell(
command = command,
outputs = [output_dir]
)
return [DefaultInfo(files = depset([output_dir])),]
my_rule = rule(
implementation = _impl,
)
usage:
$ bazel run cc
Starting local Bazel server and connecting to it...
INFO: Analyzed target //:cc (15 packages loaded, 57 targets configured).
INFO: Found 1 target...
Target //:cc up-to-date:
bazel-bin/cc
INFO: Elapsed time: 3.626s, Critical Path: 0.06s
INFO: 8 processes: 4 internal, 4 linux-sandbox.
INFO: Build completed successfully, 8 total actions
INFO: Build completed successfully, 8 total actions
hello world 42
What's the shortest shell.nix equivalent of the following cmdline arguments?
nix-shell -p "haskell.packages.ghc865.ghcWithPackages (p: [p.ghci-pretty])"
This works, but it's verbose:
# contents of shell.nix file
# run with the following cmd:
# nix-shell shell.nix
{ nixpkgs ? import <nixpkgs> {} }:
let
inherit nixpkgs;
inherit (nixpkgs) haskellPackages;
haskellDeps = a: with a; [
ipprint
base
hscolour
ghci-pretty
];
ghc = nixpkgs.haskellPackages.ghcWithPackages haskellDeps;
nixPackages = [
haskellPackages.cabal-install
ghc
];
in
nixpkgs.stdenv.mkDerivation {
name = "profile_name";
buildInputs = nixPackages;
}
You can just copy your command line verbatim like so:
{ pkgs ? import <nixpkgs> {} }:
let
ghc = pkgs.haskell.packages.ghc865.ghcWithPackages (p: [ p.ghci-pretty ]);
in
pkgs.mkShell {
buildInputs = [ ghc ];
}
I'm currently using the following to pin the nixpkgs to make a project reproducible (with https://github.com/nmattia/niv):
{
nixpkgs ? import <nixpkgs> {}
, compiler ? "default"
, doBenchmark ? false
, sources ? import ./nix/sources.nix
}:
let
niv = import sources.nixpkgs {
overlays = [
(_ : _ : { niv = import sources.niv {}; })
] ;
config = {};
};
inherit (niv) pkgs;
...
With the above, I can nix-build successfully, though I'm not 100% sure it's using niv to pin the packages as intended.
None the less, I'm now trying reference a github project - but I run into an error (...called without required argument...)?
My attempt:
After running niv add ...:
nix-shell
error: 'f' at /home/chris/fromLaptopt/usbflash/Haskell/UraniumZuluGooseberry/shell.nix:18:7 called without required argument 'platinumpitanga', at /nix/store/j6spkp2a2sqd65db1sj9zzpgrfnkrwrp-source/pkgs/development/haskell-modules/make-package-set.nix:87:27
Here is the entire shell.nix I'm using. Please note I do realize I'll still need to 'override' the Haskell / GHC package - please ignore this - I'm just hoping I need to figure out how to reference the external github package for now.
{
nixpkgs ? import <nixpkgs> {}
, compiler ? "default"
, doBenchmark ? false
, sources ? import ./nix/sources.nix
}:
let
niv = import sources.nixpkgs {
overlays = [
(_ : _ : { niv = import sources.niv {}; })
] ;
config = {};
};
inherit (niv) pkgs;
f = { mkDerivation, aeson, base, bytestring, containers, hpack
, influxdb, lens, platinumpitanga, pretty-simple, split, stdenv
, stm, string-conversions, text, time, vector
}:
mkDerivation {
pname = "UraniumZuluGooseberry";
version = "0.1.0.0";
src = ./.;
isLibrary = false;
isExecutable = true;
libraryToolDepends = [ hpack ];
executableHaskellDepends = [
aeson base bytestring containers influxdb lens platinumpitanga
pretty-simple split stm string-conversions text time vector
];
preConfigure = "hpack";
license = stdenv.lib.licenses.bsd3;
};
haskellPackages = if compiler == "default"
then pkgs.haskellPackages
else pkgs.haskell.packages.${compiler};
variant = if doBenchmark then pkgs.haskell.lib.doBenchmark else pkgs.lib.id;
drv = variant (haskellPackages.callPackage f {});
in
if pkgs.lib.inNixShell then drv.env else drv
niv = import sources.nixpkgs {
overlays = [
(_ : _ : {
niv = import sources.niv {};
platinumpitanga = niv.pkgs.haskellPackages.callCabal2nix "platinumpitanga"
(sources.PlatinumPitanga) {} ;
})
] ;
config = {};
};
inherit (niv) pkgs;
I think the above is all that is needed (I've managed to go further with the nix-build at least).
Essentially sources.PlatinumPitanga - it's the name of the "repository".
The actual "nix derivation" seems to be sources.PlatinumPitanga for example. However one still needs to import sources.PlatinumPitanga {} to use it (instantiate it?) as a "package".
It seems that the genrule can only output a Target, and the expand_template substitutions accept only string_dict, how can I use the genrule output to expand_template?
gen.bzl
def _expand_impl(ctx):
ctx.actions.expand_template(
template = ctx.file._template,
output = ctx.outputs.source_file,
substitutions = {
"{version}": ctx.attr.version,
}
)
expand = rule(
implementation = _expand_impl,
attrs = {
"version": attr.string(mandatory = True),
"_template": attr.label(
default = Label("//version:local.go.in"),
allow_single_file = True,
),
},
outputs = {"source_file": "local.go"},
)
BUILD
load("#io_bazel_rules_go//go:def.bzl", "go_library")
filegroup(
name = "templates",
srcs = ["local.go.in"],
)
genrule(
name = "inject",
outs = ["VERSION"],
local = 1,
cmd = "git rev-parse HEAD",
)
load(":gen.bzl", "expand")
expand(
name = "expand",
version = ":inject",
)
go_library(
name = "go_default_library",
srcs = [
"default.go",
":expand", # Keep
],
importpath = "go.megvii-inc.com/brain/data/version",
visibility = ["//visibility:public"],
)
and the local.go.in
package version
func init() {
V = "{version}"
}
I expect the {version} in local.go.in can be replace by git rev-parse HEAD output.
The problem here is that the substitutions argument of ctx.actions.expand_template() must be known during the analysis phase (i.e., when _expand_impl is run), which happens before the git rev-parse HEAD command of the genrule would be run (i.e., during the execution phase).
There are a few ways to do this. The simplest is to do everything in the genrule:
genrule(
name = "gen_local_go",
srcs = ["local.go.in"],
outs = ["local.go"],
local = 1,
cmd = 'sed "s/{VERSION}/$(git rev-parse HEAD)/" "$<" > "$#"',
)
That relies on sed being available on the host machine, but any other sort of program that can input one file, modify the text, and output it to another file will work.
Another option is to use a combination of --workspace_status_command
There are more details here:
How to run a shell command at analysis time in bazel?
The advantage of this approach is that it avoids local genrules.