I'm trying to write a program that from the main "server", calls for 2 "agents" that each one of them creates containers. Each container should run periodically and every 10 seconds print the time. The create command is taken from the Std.in as <command> <PATH>.
For example, I pass 4 containers to create, so agent1 creates 2, and agent2 creates 2.
My problem is that when I run this program I only see the second container from agent1 when I type docker ps I get only mycontainer2.
And only after I stop/remove mycontainer2, only then agent2 creates mycontainer4. (mycontainer1 and mycontainer3 wasn't created at all).
Probably the way I created them was wrong. Is there any way I can create all of them and let them run separately?
Main "server":
package main
import (
"fmt"
"gopkg.in/yaml.v2"
"io/ioutil"
"log"
"bufio"
"os"
"bytes"
"os/exec"
"strings"
"strconv"
)
type conf struct {
Name string `yaml:"Name"`
Amount int `yaml:"Amount"`
Image string `yaml:"Image"`
}
func (c *conf) getConf() *conf {
yamlFile, err := ioutil.ReadFile("conf.yaml")
if err != nil {
log.Printf("yamlFile.Get err #%v ", err)
}
err = yaml.Unmarshal(yamlFile, c)
if err != nil {
log.Fatalf("Unmarshal: %v", err)
}
return c
}
func isError(err error) bool {
if err != nil {
fmt.Println(err.Error())
}
return (err != nil)
}
func showEnvStatus() {
cmd := exec.Command("echo", "Hello from ShowEnv")
// cmd.Stdin = strings.NewReader("some input")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
fmt.Printf("The out put is: %q\n", out.String())
return
}
func createAgents(c conf){
var workload = c.Amount/2
var rightBroderAgent1 = strconv.Itoa(workload)
var leftBorderAgent2 = strconv.Itoa(c.Amount- workload + 1)
//invoke agent1. Args are the range of the containers numbers for names. (0 : workload)
cmd1 := exec.Command("./agent","1", rightBroderAgent1, c.Name, c.Image)
var out bytes.Buffer
cmd1.Stdout = &out
err1 := cmd1.Run()
if err1 != nil {
fmt.Println(err1)
}
fmt.Println("%q\n",out.String())
//invoke agent2. Args are the range of the containers numbers for names.(workload+1 : Amount)
cmd2 := exec.Command("./agent",leftBorderAgent2, strconv.Itoa(c.Amount),c.Name, c.Image)
err2 := cmd2.Run()
if err2 != nil {
log.Fatal(err2)
}
return
}
func parseCommand(text string) ([]string){
res := strings.Split(text, " ")
return res
}
func main() {
i := 0
for i < 1{
var args []string
reader := bufio.NewReader(os.Stdin)
text, _ := reader.ReadString('\n')
args = parseCommand(text)
switch args[0] {
case "create":
var c conf
c.getConf()
createAgents(c)
default:
fmt.Println("Unknown command, try again")
}
}
}
############################# Agent file ##################
func main(){
// Args[] structure : {FILE, left border, right border, container's name, image}
left,_ := strconv.Atoi(os.Args[1])
right,_ := strconv.Atoi(os.Args[2])
for i := left; i <= right; i++ {
fmt.Println(left, right)
var name string = os.Args[3] + strconv.Itoa(i+1)
cmd1 := exec.Command("docker", "run", "--name", name, "-it", "-d", os.Args[4])
cmd2 := exec.Command("docker", "cp", "showtime" ,name + ":/showtime")
cmd3 := exec.Command("docker", "exec" ,name , "./showtime")
// cmds := []*exec.Cmd{cmd1,cmd2,cmd3}
err1 := cmd1.Run()
if err1 != nil {
fmt.Println(err1)
os.Exit(2)
}
err2 := cmd2.Run()
if err2 != nil {
fmt.Println(err2)
os.Exit(2)
}
err3 := cmd3.Run()
if err3 != nil {
fmt.Println(err3)
os.Exit(2)
}
}
}
Related
I have a requirement to fetch docker container's logs, I'm using below code to fetch docker logs,
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
panic(err)
}
options := types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Since: "",
Until: "",
Timestamps: false,
Follow: true,
Tail: "",
Details: true,
}
out, err := cli.ContainerLogs(ctx, "bcd693465a62", options)
if err != nil {
panic(err)
}
buf := new(strings.Builder)
_, err = io.Copy(buf, out)
if err != nil {
fmt.Println(err)
}
fmt.Printf("%s", buf)
My problem is I'm getting docker logs in two different format, for some containers it's just plain text
exec /entrypoint.sh: no such file or directory
but in containerID-json.log file it's showing in below format
{"log":"exec /entrypoint.sh: no such file or directory\n","stream":"stderr","time":"2022-09-06T12:23:57.741316145Z"}
and for some containers it's showing in dynamic format/schema like
time="2022-09-05T02:13:44Z" level=debug msg="cleanup aborting ingest" ref="buildkit/1/layer-sha256:2774afd0c4d3ded992c58f3b5e5939d091bd26f40e507c6dc21dcbd8b7ff486f"
time="2022-09-05T02:13:44Z" level=debug msg="content garbage collected" d=2.971574ms
How I can collect all docker container's logs in same schema/format so it can be stored in below JSON format?
{
"containerID": "bcd693465a62",
"logs": [
{"log":"exec /entrypoint.sh: no such file or directory\n","stream":"stderr","time":"2022-09-06T12:23:57.741316145Z"},
{"log":"cleanup aborting ingest","stream":"stdout","time":"2022-09-05T02:13:44Z"}
]
}
Any help is appreciated
I wrote below code with the help of code reference by #BrianWagner to fulfill my requirements (It works as expected), please suggest if it can be done better way, or if any improvements.
package main
import (
"context"
"encoding/binary"
"fmt"
"io"
"log"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
)
type DockerContainerLog struct {
ContainerID string `json:"containerID"`
DockerLogs []DockerLog `json:"dockerLogs"`
}
type DockerLog struct {
Time string `json:"time"`
Stream string `json:"stream"`
Log string `json:"log"`
}
func main() {
logs := getContainerLogs("cd3d8362ba45")
fmt.Print(logs)
}
func getContainerLogs(cid string) DockerContainerLog {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
panic(err)
}
options := types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Since: "",
Until: "",
Timestamps: true,
Follow: true,
Tail: "",
Details: false,
}
reader, err := cli.ContainerLogs(context.Background(), cid, options)
if err != nil {
log.Fatal(err)
}
defer reader.Close()
dockerLogs := DockerContainerLog{ContainerID: cid}
hdr := make([]byte, 8)
for {
var docLog DockerLog
_, err := reader.Read(hdr)
if err != nil {
if err == io.EOF {
return dockerLogs
}
log.Fatal(err)
}
count := binary.BigEndian.Uint32(hdr[4:])
dat := make([]byte, count)
_, err = reader.Read(dat)
if err != nil && err != io.EOF {
log.Fatal(err)
}
time, log, found := strings.Cut(string(dat), " ")
if found {
docLog.Time = time
docLog.Log = log
switch hdr[0] {
case 1:
docLog.Stream = "Stdout"
default:
docLog.Stream = "Stderr"
}
dockerLogs.DockerLogs = append(dockerLogs.DockerLogs, docLog)
}
}
}
Need help.I have code to exec command from docker container. Need corrently get stdout from exec command.
execConfig:= types.ExecConfig{Tty:false,AttachStdout:true,AttachStderr:false,Cmd:command}
respIdExecCreate,err := cli.ContainerExecCreate(context.Background(),dockerName,execConfig)
if err != nil {
fmt.Println(err)
}
respId,err:=cli.ContainerExecAttach(context.Background(),respIdExecCreate.ID,types.ExecStartCheck{})
if err != nil {
fmt.Println(err)
}
scanner := bufio.NewScanner(respId.Reader)
for scanner.Scan() {
fmt.Println(output)
}
From output i see interesting situation:
Screen from gyazo
How corrently remove bytes ?
I send simply command := []string{"echo","-n", "hello word"}
I've faced with same issue, this is how stderr and stdout looks for me:
StdOut: "\x01\x00\x00\x00\x00\x00\x00\thello world\n"
StdErr: "\x01\x00\x00\x00\x00\x00\x00fError: Exec command has already run\r\n"
I've cheched docker source code and found answer here:
https://github.com/moby/moby/blob/8e610b2b55bfd1bfa9436ab110d311f5e8a74dcb/integration/internal/container/exec.go#L38
looks like this leading bytes used especially for marking stdout and stderr bytes.
And there is a library "github.com/docker/docker/pkg/stdcopy" which can split stdout and stderr from stream reader:
type ExecResult struct {
StdOut string
StdErr string
ExitCode int
}
func Exec(ctx context.Context, containerID string, command []string) (types.IDResponse, error) {
docker, err := client.NewEnvClient()
if err != nil {
return types.IDResponse{}, err
}
defer closer(docker)
config := types.ExecConfig{
AttachStderr: true,
AttachStdout: true,
Cmd: command,
}
return docker.ContainerExecCreate(ctx, containerID, config)
}
func InspectExecResp(ctx context.Context, id string) (ExecResult, error) {
var execResult ExecResult
docker, err := client.NewEnvClient()
if err != nil {
return execResult, err
}
defer closer(docker)
resp, err := docker.ContainerExecAttach(ctx, id, types.ExecConfig{})
if err != nil {
return execResult, err
}
defer resp.Close()
// read the output
var outBuf, errBuf bytes.Buffer
outputDone := make(chan error)
go func() {
// StdCopy demultiplexes the stream into two buffers
_, err = stdcopy.StdCopy(&outBuf, &errBuf, resp.Reader)
outputDone <- err
}()
select {
case err := <-outputDone:
if err != nil {
return execResult, err
}
break
case <-ctx.Done():
return execResult, ctx.Err()
}
stdout, err := ioutil.ReadAll(&outBuf)
if err != nil {
return execResult, err
}
stderr, err := ioutil.ReadAll(&errBuf)
if err != nil {
return execResult, err
}
res, err := docker.ContainerExecInspect(ctx, id)
if err != nil {
return execResult, err
}
execResult.ExitCode = res.ExitCode
execResult.StdOut = string(stdout)
execResult.StdErr = string(stderr)
return execResult, nil
}
I want make a web terminal by kubernetes client-go api, and now i meet some problem.
main:
func main() {
flag.Parse()
log.SetFlags(0)
http.HandleFunc("/ws", echo)
log.Fatal(http.ListenAndServe(*addr, nil))
}
struct&handleFunc:
type Cmd struct {
Conn *websocket.Conn
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
func echo(w http.ResponseWriter, r *http.Request) {
cmd := Cmd{}
cmd.Stderr = ioutil.Discard
c, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
cmd.Conn = c
cmd.handleClient()
}
func (c *Cmd) handleClient() {
//c.Stdout = os.Stdout
//c.Stdin = os.Stdin
var read io.Reader
var write io.Writer
c.Stdin = read
c.Stdout = write
go func(rd io.Reader) {
for {
_, r, err := c.Conn.NextReader()
if err != nil {
fmt.Println("ReadErr:", err)
c.Conn.Close()
return
}
rd = r
}
}(read)
go func(wr io.Writer) {
w, err := c.Conn.NextWriter(1)
if err != nil {
fmt.Println("WriteErr:", err)
c.Conn.Close()
return
}
wr = w
}(write)
err := api.ExecOperator{}.ExecConsoleInContainer("1", "nginx-65899c769f-mkswf", c.Stdin, c.Stdout, c.Stderr)
if err != nil {
fmt.Println("ExecErr:", err)
return
}
}
api.ExecOperator{}.ExecConsoleInContainer() is a func invoke kubernetes exec api like kubernetes exec_util.go
When i use the code below,i can operate at the console,but i dont know how to use websocket(gorilla/websocket) input and output to replace os.stdin and os.stdout,i tried to write some of the code above ,but it cant work.
c.Stdout = os.Stdout
c.Stdin = os.Stdin
I wanted to know how the container id is generated so please provide the source code that provides the container id when the docker run is executed?
Here is a code snippet from docker daemon's function for creating Containers:
func (daemon *Daemon) newContainer(name string, config *runconfig.Config, imgID string) (*Container, error) {
var (
id string
err error
)
id, name, err = daemon.generateIDAndName(name)
if err != nil {
return nil, err
}
…
base := daemon.newBaseContainer(id)
…
base.ExecDriver = daemon.execDriver.Name()
return &base, err
}
So, the logic of creating ID and Name is in generateIDAndName function:
func (daemon *Daemon) generateIDAndName(name string) (string, string, error) {
var (
err error
id = stringid.GenerateNonCryptoID()
)
if name == "" {
if name, err = daemon.generateNewName(id); err != nil {
return "", "", err
}
return id, name, nil
}
if name, err = daemon.reserveName(id, name); err != nil {
return "", "", err
}
return id, name, nil
}
Here is stringid sources and the concrete method is generateID with false as input parameter:
func generateID(crypto bool) string {
b := make([]byte, 32)
var r io.Reader = random.Reader
if crypto {
r = rand.Reader
}
for {
if _, err := io.ReadFull(r, b); err != nil {
panic(err) // This shouldn't happen
}
id := hex.EncodeToString(b)
// if we try to parse the truncated for as an int and we don't have
// an error then the value is all numberic and causes issues when
// used as a hostname. ref #3869
if _, err := strconv.ParseInt(TruncateID(id), 10, 64); err == nil {
continue
}
return id
}
}
As you can see, the value is randomly generated with this random
// Reader is a global, shared instance of a pseudorandom bytes generator.
// It doesn't consume entropy.
var Reader io.Reader = &reader{rnd: Rand}
I'm generating some source code using the templates package( is there a better method? )and part of the testing I need to check if the output matches the expected source code.
I tried a string comparison but it fails due the extra spaces / new lines generated by the templates package. I've also tried format.Source with not success. ( FAIL)
I tried to parse the ast of the both sources (see bellow) but the ast doesn't match either even if the code is basically same except the new lines / spaces. (FAIL)
package main
import (
"fmt"
"go/parser"
"go/token"
"reflect"
)
func main() {
stub1 := `package main
func myfunc(s string) error {
return nil
}`
stub2 := `package main
func myfunc(s string) error {
return nil
}`
fset := token.NewFileSet()
r1, err := parser.ParseFile(fset, "", stub1, parser.AllErrors)
if err != nil {
panic(err)
}
fset = token.NewFileSet()
r2, err := parser.ParseFile(fset, "", stub2, parser.AllErrors)
if err != nil {
panic(err)
}
if !reflect.DeepEqual(r1, r2) {
fmt.Printf("e %v, r %s, ", r1, r2)
}
}
Playground
Well, one simple way to achieve this is to use the go/printer library, that gives you better control of output formatting, and is basically like running gofmt on the source, normalizing both trees:
package main
import (
"fmt"
"go/parser"
"go/token"
"go/printer"
//"reflect"
"bytes"
)
func main() {
stub1 := `package main
func myfunc(s string) error {
return nil
}`
stub2 := `package main
func myfunc(s string) error {
return nil
}`
fset1 := token.NewFileSet()
r1, err := parser.ParseFile(fset1, "", stub1, parser.AllErrors)
if err != nil {
panic(err)
}
fset2 := token.NewFileSet()
r2, err := parser.ParseFile(fset1, "", stub2, parser.AllErrors)
if err != nil {
panic(err)
}
// we create two output buffers for each source tree
out1 := bytes.NewBuffer(nil)
out2 := bytes.NewBuffer(nil)
// we use the same printer config for both
conf := &printer.Config{Mode: printer.TabIndent | printer.UseSpaces, Tabwidth: 8}
// print to both outputs
if err := conf.Fprint(out1, fset1, r1); err != nil {
panic(err)
}
if err := conf.Fprint(out2, fset2, r2); err != nil {
panic(err)
}
// they should be identical!
if string(out1.Bytes()) != string(out2.Bytes()) {
panic(string(out1.Bytes()) +"\n" + string(out2.Bytes()))
} else {
fmt.Println("A-OKAY!")
}
}
Of course this code needs to be refactored to not look as stupid. Another approach is instead of using DeepEqual, create a tree comparison function yourself, that skips irrelevant nodes.
This was easier than I thought. All I had to do was to remove the empty new lines(after formatting). Below is the code.
package main
import (
"fmt"
"go/format"
"strings"
)
func main() {
a, err := fmtSource(stub1)
if err != nil {
panic(err)
}
b, err := fmtSource(stub2)
if err != nil {
panic(err)
}
if a != b {
fmt.Printf("a %v, \n b %v", a, b)
}
}
func fmtSource(source string) (string, error) {
if !strings.Contains(source, "package") {
source = "package main\n" + source
}
b, err := format.Source([]byte(source))
if err != nil {
return "", err
}
// cleanLine replaces double space with one space
cleanLine := func(s string)string{
sa := strings.Fields(s)
return strings.Join(sa, " ")
}
lines := strings.Split(string(b), "\n")
n := 0
var startLn *int
for _, line := range lines {
if line != "" {
line = cleanLine(line)
lines[n] = line
if startLn == nil {
x := n
startLn = &x
}
n++
}
}
lines = lines[*startLn:n]
// Add final "" entry to get trailing newline from Join.
if n > 0 && lines[n-1] != "" {
lines = append(lines, "")
}
// Make it pretty
b, err = format.Source([]byte(strings.Join(lines, "\n")))
if err != nil {
return "", err
}
return string(b), nil
}