How can I compare two source code files/ ast trees? - parsing

I'm generating some source code using the templates package( is there a better method? )and part of the testing I need to check if the output matches the expected source code.
I tried a string comparison but it fails due the extra spaces / new lines generated by the templates package. I've also tried format.Source with not success. ( FAIL)
I tried to parse the ast of the both sources (see bellow) but the ast doesn't match either even if the code is basically same except the new lines / spaces. (FAIL)
package main
import (
"fmt"
"go/parser"
"go/token"
"reflect"
)
func main() {
stub1 := `package main
func myfunc(s string) error {
return nil
}`
stub2 := `package main
func myfunc(s string) error {
return nil
}`
fset := token.NewFileSet()
r1, err := parser.ParseFile(fset, "", stub1, parser.AllErrors)
if err != nil {
panic(err)
}
fset = token.NewFileSet()
r2, err := parser.ParseFile(fset, "", stub2, parser.AllErrors)
if err != nil {
panic(err)
}
if !reflect.DeepEqual(r1, r2) {
fmt.Printf("e %v, r %s, ", r1, r2)
}
}
Playground

Well, one simple way to achieve this is to use the go/printer library, that gives you better control of output formatting, and is basically like running gofmt on the source, normalizing both trees:
package main
import (
"fmt"
"go/parser"
"go/token"
"go/printer"
//"reflect"
"bytes"
)
func main() {
stub1 := `package main
func myfunc(s string) error {
return nil
}`
stub2 := `package main
func myfunc(s string) error {
return nil
}`
fset1 := token.NewFileSet()
r1, err := parser.ParseFile(fset1, "", stub1, parser.AllErrors)
if err != nil {
panic(err)
}
fset2 := token.NewFileSet()
r2, err := parser.ParseFile(fset1, "", stub2, parser.AllErrors)
if err != nil {
panic(err)
}
// we create two output buffers for each source tree
out1 := bytes.NewBuffer(nil)
out2 := bytes.NewBuffer(nil)
// we use the same printer config for both
conf := &printer.Config{Mode: printer.TabIndent | printer.UseSpaces, Tabwidth: 8}
// print to both outputs
if err := conf.Fprint(out1, fset1, r1); err != nil {
panic(err)
}
if err := conf.Fprint(out2, fset2, r2); err != nil {
panic(err)
}
// they should be identical!
if string(out1.Bytes()) != string(out2.Bytes()) {
panic(string(out1.Bytes()) +"\n" + string(out2.Bytes()))
} else {
fmt.Println("A-OKAY!")
}
}
Of course this code needs to be refactored to not look as stupid. Another approach is instead of using DeepEqual, create a tree comparison function yourself, that skips irrelevant nodes.

This was easier than I thought. All I had to do was to remove the empty new lines(after formatting). Below is the code.
package main
import (
"fmt"
"go/format"
"strings"
)
func main() {
a, err := fmtSource(stub1)
if err != nil {
panic(err)
}
b, err := fmtSource(stub2)
if err != nil {
panic(err)
}
if a != b {
fmt.Printf("a %v, \n b %v", a, b)
}
}
func fmtSource(source string) (string, error) {
if !strings.Contains(source, "package") {
source = "package main\n" + source
}
b, err := format.Source([]byte(source))
if err != nil {
return "", err
}
// cleanLine replaces double space with one space
cleanLine := func(s string)string{
sa := strings.Fields(s)
return strings.Join(sa, " ")
}
lines := strings.Split(string(b), "\n")
n := 0
var startLn *int
for _, line := range lines {
if line != "" {
line = cleanLine(line)
lines[n] = line
if startLn == nil {
x := n
startLn = &x
}
n++
}
}
lines = lines[*startLn:n]
// Add final "" entry to get trailing newline from Join.
if n > 0 && lines[n-1] != "" {
lines = append(lines, "")
}
// Make it pretty
b, err = format.Source([]byte(strings.Join(lines, "\n")))
if err != nil {
return "", err
}
return string(b), nil
}

Related

Is there any way to run my containers periodically?

I'm trying to write a program that from the main "server", calls for 2 "agents" that each one of them creates containers. Each container should run periodically and every 10 seconds print the time. The create command is taken from the Std.in as <command> <PATH>.
For example, I pass 4 containers to create, so agent1 creates 2, and agent2 creates 2.
My problem is that when I run this program I only see the second container from agent1 when I type docker ps I get only mycontainer2.
And only after I stop/remove mycontainer2, only then agent2 creates mycontainer4. (mycontainer1 and mycontainer3 wasn't created at all).
Probably the way I created them was wrong. Is there any way I can create all of them and let them run separately?
Main "server":
package main
import (
"fmt"
"gopkg.in/yaml.v2"
"io/ioutil"
"log"
"bufio"
"os"
"bytes"
"os/exec"
"strings"
"strconv"
)
type conf struct {
Name string `yaml:"Name"`
Amount int `yaml:"Amount"`
Image string `yaml:"Image"`
}
func (c *conf) getConf() *conf {
yamlFile, err := ioutil.ReadFile("conf.yaml")
if err != nil {
log.Printf("yamlFile.Get err #%v ", err)
}
err = yaml.Unmarshal(yamlFile, c)
if err != nil {
log.Fatalf("Unmarshal: %v", err)
}
return c
}
func isError(err error) bool {
if err != nil {
fmt.Println(err.Error())
}
return (err != nil)
}
func showEnvStatus() {
cmd := exec.Command("echo", "Hello from ShowEnv")
// cmd.Stdin = strings.NewReader("some input")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
fmt.Printf("The out put is: %q\n", out.String())
return
}
func createAgents(c conf){
var workload = c.Amount/2
var rightBroderAgent1 = strconv.Itoa(workload)
var leftBorderAgent2 = strconv.Itoa(c.Amount- workload + 1)
//invoke agent1. Args are the range of the containers numbers for names. (0 : workload)
cmd1 := exec.Command("./agent","1", rightBroderAgent1, c.Name, c.Image)
var out bytes.Buffer
cmd1.Stdout = &out
err1 := cmd1.Run()
if err1 != nil {
fmt.Println(err1)
}
fmt.Println("%q\n",out.String())
//invoke agent2. Args are the range of the containers numbers for names.(workload+1 : Amount)
cmd2 := exec.Command("./agent",leftBorderAgent2, strconv.Itoa(c.Amount),c.Name, c.Image)
err2 := cmd2.Run()
if err2 != nil {
log.Fatal(err2)
}
return
}
func parseCommand(text string) ([]string){
res := strings.Split(text, " ")
return res
}
func main() {
i := 0
for i < 1{
var args []string
reader := bufio.NewReader(os.Stdin)
text, _ := reader.ReadString('\n')
args = parseCommand(text)
switch args[0] {
case "create":
var c conf
c.getConf()
createAgents(c)
default:
fmt.Println("Unknown command, try again")
}
}
}
############################# Agent file ##################
func main(){
// Args[] structure : {FILE, left border, right border, container's name, image}
left,_ := strconv.Atoi(os.Args[1])
right,_ := strconv.Atoi(os.Args[2])
for i := left; i <= right; i++ {
fmt.Println(left, right)
var name string = os.Args[3] + strconv.Itoa(i+1)
cmd1 := exec.Command("docker", "run", "--name", name, "-it", "-d", os.Args[4])
cmd2 := exec.Command("docker", "cp", "showtime" ,name + ":/showtime")
cmd3 := exec.Command("docker", "exec" ,name , "./showtime")
// cmds := []*exec.Cmd{cmd1,cmd2,cmd3}
err1 := cmd1.Run()
if err1 != nil {
fmt.Println(err1)
os.Exit(2)
}
err2 := cmd2.Run()
if err2 != nil {
fmt.Println(err2)
os.Exit(2)
}
err3 := cmd3.Run()
if err3 != nil {
fmt.Println(err3)
os.Exit(2)
}
}
}

kubernetes attach pod use websocket

I want make a web terminal by kubernetes client-go api, and now i meet some problem.
main:
func main() {
flag.Parse()
log.SetFlags(0)
http.HandleFunc("/ws", echo)
log.Fatal(http.ListenAndServe(*addr, nil))
}
struct&handleFunc:
type Cmd struct {
Conn *websocket.Conn
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
func echo(w http.ResponseWriter, r *http.Request) {
cmd := Cmd{}
cmd.Stderr = ioutil.Discard
c, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
cmd.Conn = c
cmd.handleClient()
}
func (c *Cmd) handleClient() {
//c.Stdout = os.Stdout
//c.Stdin = os.Stdin
var read io.Reader
var write io.Writer
c.Stdin = read
c.Stdout = write
go func(rd io.Reader) {
for {
_, r, err := c.Conn.NextReader()
if err != nil {
fmt.Println("ReadErr:", err)
c.Conn.Close()
return
}
rd = r
}
}(read)
go func(wr io.Writer) {
w, err := c.Conn.NextWriter(1)
if err != nil {
fmt.Println("WriteErr:", err)
c.Conn.Close()
return
}
wr = w
}(write)
err := api.ExecOperator{}.ExecConsoleInContainer("1", "nginx-65899c769f-mkswf", c.Stdin, c.Stdout, c.Stderr)
if err != nil {
fmt.Println("ExecErr:", err)
return
}
}
api.ExecOperator{}.ExecConsoleInContainer() is a func invoke kubernetes exec api like kubernetes exec_util.go
When i use the code below,i can operate at the console,but i dont know how to use websocket(gorilla/websocket) input and output to replace os.stdin and os.stdout,i tried to write some of the code above ,but it cant work.
c.Stdout = os.Stdout
c.Stdin = os.Stdin

How to parse the statements in a method or function with Go

I would like to parse go code, and more especially the content of a function.
So far it's pretty easy to get the function declaration with the parser.ParseFile function.
package main
import (
"fmt"
"go/parser"
"go/token"
"go/ast"
"log"
)
var code = []byte(`
package main
import (
"fmt"
)
func GetFoo() {
test := foo()
fmt.Println(test)
}
func foo() int {
return 0
}
`)
func main() {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "", code, parser.ParseComments)
if err != nil {
log.Fatal(err)
}
for _, decl := range f.Decls {
switch t := decl.(type) {
// That's a func decl !
case *ast.FuncDecl:
fmt.Printf("Function name: %v\n", t.Name)
// Now I would like to access to the test var and get its type
// Must be int, because foo() returns an int
}
}
}
Now I would like to access to the test var and get its type, but I am a little bit lost. I have seen the decl.Body.List to iterate over the statements but I can get that my test var is an int
Thank you for your precious help !
https://play.golang.org/p/Y8uwM-CDWy
Thanks to JimB and his hint about go/types, here's how I could get the signature of my method
package main
import (
"fmt"
"go/ast"
"go/importer"
"go/parser"
"go/token"
"go/types"
"log"
)
var code = []byte(`
package main
import (
"fmt"
)
func GetFoo() {
test := foo()
fmt.Println(test)
}
func foo() int {
return 0
}
`)
func main() {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, "", code, parser.ParseComments)
if err != nil {
log.Fatal(err)
}
conf := types.Config{Importer: importer.Default()}
pkg, err := conf.Check("cmd", fset, []*ast.File{f}, nil)
scope := pkg.Scope()
for _, decl := range f.Decls {
switch t := decl.(type) {
// That's a func decl !
case *ast.FuncDecl:
for _, s := range t.Body.List {
switch as := s.(type) {
case *ast.AssignStmt:
for _, l := range as.Rhs {
switch rh := l.(type) {
case *ast.CallExpr:
if i, ok := rh.Fun.(*ast.Ident); ok {
ft := scope.Lookup(i.Name)
if ft != nil {
if ftype, ok := ft.(*types.Func); ok {
f := ftype.Type()
if sig, ok := f.(*types.Signature); ok {
// get the returned elements
r := sig.Results()
for i := 0; i < r.Len(); i++ {
v := r.At(i)
varTpe := v.Type()
fmt.Println(varTpe)
}
}
}
}
}
}
}
}
}
}
}
}

How to flatten message with mutable structure into protobuf?

My proto buf format is this:
package main;
message Test {
optional string id = 1;
optional string name = 2;
optional string age = 3;
}
Then I populate the protobuf files from the input in golang using the following code. str is already parsed.
test = &Test{
id: proto.String(str[0]),
name: proto.String(str[1]),
age: proto.String(str[2]),
},
One condition I have to handle is that one or more optional fields in the Test structure could be absent randomly and I do not know that in advance. How do I handle that in golang?
To give more context, the real data can look like these in the file:
id=1, name=peter, age=24
id=2, age=30
name=mary, age=31
id=100
name=bob
age=11
You could use a regular expression to change your input strings into valid JSON, the use the encoding/json package to parse it. This has the advantage of letting the json parser take care of everything for you. Here is the regex for your particular case.
Basically, the regex looks for field=value and replaces with "field" : "value" and wraps it in {} to create valid JSON. The commas are left as is.
https://play.golang.org/p/_EEdTB6sve
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"regexp"
)
var ins = []string{
`id=1, name=peter, age=24`,
`id=2, age=30`,
`name=mary, age=31`,
`id=100`,
`name=bob`,
`age=11`,
}
var ParseError = errors.New("bad parser input")
var Regex *regexp.Regexp
type Test struct {
ID string
Name string
Age string
}
func (t *Test) String() string {
return fmt.Sprintf("ID: %s, Name: %s, Age: %s", t.ID, t.Name, t.Age)
}
func main() {
var err error
Regex, err = regexp.Compile(`([^,\s]*)=([^,\s]*)`)
if err != nil {
log.Panic(err)
}
for _, v := range ins {
test, err := ParseLine(v)
if err != nil {
log.Panic(err)
}
fmt.Println(test)
}
}
func ParseLine(inp string) (*Test, error) {
JSON := fmt.Sprintf("{%s}", Regex.ReplaceAllString(inp, `"$1" : "$2"`))
test := &Test{}
err := json.Unmarshal([]byte(JSON), test)
if err != nil {
return nil, err
}
return test, nil
}
Here is what I believe to be a minimum working case for what you are after, though I am not familiar enough with protocol buffers to get the strings to print right... or even verify if they are correct. Note that this doesn't run in the playground.
package main
import (
"errors"
"fmt"
"log"
"regexp"
"github.com/golang/protobuf/jsonpb"
_ "github.com/golang/protobuf/proto"
)
var ins = []string{
`id=1, name=peter, age=24`,
`id=2, age=30`,
`name=mary, age=31`,
`id=100`,
`name=bob`,
`age=11`,
}
var ParseError = errors.New("bad parser input")
var Regex *regexp.Regexp
type Test struct {
ID *string `protobuf:"bytes,1,opt,name=id,json=id" json:"id,omitempty"`
Name *string `protobuf:"bytes,2,opt,name=name,json=name" json:"name,omitempty"`
Age *string `protobuf:"bytes,3,opt,name=age,json=age" json:"age,omitempty"`
}
func (t *Test) Reset() {
*t = Test{}
}
func (*Test) ProtoMessage() {}
func (*Test) Descriptor() ([]byte, []int) {return []byte{}, []int{0}}
func (t *Test) String() string {
return fmt.Sprintf("ID: %v, Name: %v, Age: %v", t.ID, t.Name, t.Age)
}
func main() {
var err error
Regex, err = regexp.Compile(`([^,\s]*)=([^,\s]*)`)
if err != nil {
log.Panic(err)
}
for _, v := range ins {
test, err := ParseLine(v)
if err != nil {
fmt.Println(err)
log.Panic(err)
}
fmt.Println(test)
}
}
func ParseLine(inp string) (*Test, error) {
JSON := fmt.Sprintf("{%s}", Regex.ReplaceAllString(inp, `"$1" : "$2"`))
test := &Test{}
err := jsonpb.UnmarshalString(JSON, test)
if err != nil {
return nil, err
}
return test, nil
}
Looks like you can write the parser for each line of your input something like the following.
NOTE: I didn't make the struct with proto values because as an external package, it can't be imported in the playground.
https://play.golang.org/p/hLZvbiMMlZ
package main
import (
"errors"
"fmt"
"strings"
)
var ins = []string{
`id=1, name=peter, age=24`,
`id=2, age=30`,
`name=mary, age=31`,
`id=100`,
`name=bob`,
`age=11`,
}
var ParseError = errors.New("bad parser input")
type Test struct {
ID string
Name string
Age string
}
func (t *Test) String() string {
return fmt.Sprintf("ID: %s, Name: %s, Age: %s", t.ID, t.Name, t.Age)
}
func main() {
for _, v := range ins {
t, err := ParseLine(v)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(t)
}
}
}
func ParseLine(inp string) (*Test, error) {
splt := strings.Split(inp, ",")
test := &Test{}
for _, f := range splt {
fieldVal := strings.Split(strings.TrimSpace(f), "=")
switch strings.TrimSpace(fieldVal[0]) {
case "id":
test.ID = strings.TrimSpace(fieldVal[1])
case "name":
test.Name = strings.TrimSpace(fieldVal[1])
case "age":
test.Age = strings.TrimSpace(fieldVal[1])
default:
return nil, ParseError
}
}
return test, nil
}

How to parse a method declaration?

I'm trying to parse a method declaration. Basically I need to get the syntax node of the receiver base type (type hello) and the return types (notype and error). The ast package seems straightforward but for some reason I don't get the data I need (i.e. the fields are reported nil).
The only useful data seems provided only in Object -> Decl field which is of type interface{} so I don't think I can serialize it.
Any help would be appreciated. Code below:
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
)
func main() {
// src is the input for which we want to inspect the AST.
src := `
package mypack
// type hello is a cool type
type hello string
// type notype is not that cool
type notype int
// func printme is like nothing else.
func (x *hello)printme(s string)(notype, error){
return 0, nil
}
`
// Create the AST by parsing src.
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, "src.go", src, 0)
if err != nil {
panic(err)
}
// Inspect the AST and find our function
var mf ast.FuncDecl
ast.Inspect(f, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.FuncDecl:
mf = *x
}
return true
})
if mf.Recv != nil {
fmt.Printf("\n receivers:")
for _, v := range mf.Recv.List {
fmt.Printf(",tag %v", v.Tag)
for _, xv := range v.Names {
fmt.Printf("name %v, decl %v, data %v, type %v",
xv.Name, xv.Obj.Decl, xv.Obj.Data, xv.Obj.Type)
}
}
}
}
Playground
To get the type you need to look at the Type attribute which could be an ast.StarExpr or an ast.Ident.
Here take a look at this :
package main
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
)
func main() {
// src is the input for which we want to inspect the AST.
src := `
package mypack
// type hello is a cool type
type hello string
// type notype is not that cool
type notype int
// printme is like nothing else.
func (x *hello)printme(s string)(notype, error){
return 0, nil
}
`
// Create the AST by parsing src.
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, "src.go", src, 0)
if err != nil {
panic(err)
}
// Inspect the AST and find our function
var mf ast.FuncDecl
ast.Inspect(f, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.FuncDecl:
mf = *x
}
return true
})
if mf.Recv != nil {
for _, v := range mf.Recv.List {
fmt.Print("recv type : ")
switch xv := v.Type.(type) {
case *ast.StarExpr:
if si, ok := xv.X.(*ast.Ident); ok {
fmt.Println(si.Name)
}
case *ast.Ident:
fmt.Println(xv.Name)
}
}
}
}

Resources