I have created a DroneCI pipeline with the following content:
kind: pipeline
type: docker
name: Build auto git tagger
steps:
- name: test and build
image: golang
commands:
- go mod download
- go test ./test
- go build -o ./build/package ./cmd/git-tagger
- name: Build docker image
image: plugins/docker
pull: if-not-exists
settings:
username:
password:
repo:
dockerfile:
registry:
auto_tag:
trigger:
branch:
- master
The go test starts a gogs docker container for testing purpose, here is the code:
func createGogsContainer(dest, waitUrl string) (stopContainer, error) {
client, err := docker.NewClientFromEnv()
if err != nil {
return nil, err
}
ctx := context.Background()
gogs, err := client.CreateContainer(docker.CreateContainerOptions{
Name: "repo",
Config: &docker.Config{
Image: "gogs/gogs",
},
HostConfig: &docker.HostConfig{
PublishAllPorts: true,
AutoRemove: true,
Mounts: []docker.HostMount{
{
Type: "bind",
Source: dest,
Target: "/data",
}},
PortBindings: map[docker.Port][]docker.PortBinding{
"3000/tcp": {{HostIP: "0.0.0.0", HostPort: "8888"}},
"22/tcp": {{HostIP: "0.0.0.0", HostPort: "2222"}},
},
},
Context: ctx,
})
if err != nil {
return nil, err
}
err = client.StartContainer(gogs.ID, nil)
if err != nil {
return nil, err
}
//Wait for connection
host, err := url.Parse(waitUrl)
if err != nil {
return nil, err
}
err = waitHTTP(fmt.Sprintf("%s://%s", host.Scheme, host.Host), 3, 0)
if err != nil {
return nil, err
}
return func() error {
return client.StopContainerWithContext(gogs.ID, 5, ctx)
}, nil
}
The pipeline has been aborted with following error message:
latest: Pulling from library/golang
Digest: sha256:f30b0d05ea7783131d84deea3b5f4d418d9d930dfa3668a9a5fa253d1f9dce5a
Status: Image is up to date for golang:latest
+ go mod download
+ go test ./test
time="2020-04-23T17:58:24Z" level=error msg="Get \"http://0.0.0.0:8888/gat/WithoutTag.git/info/refs?service=git-upload-pack\": dial tcp 0.0.0.0:8888: connect: connection refused"
time="2020-04-23T17:58:24Z" level=error msg="Get \"http://0.0.0.0:8888/gat/WithoutTag.git/info/refs?service=git-upload-pack\": dial tcp 0.0.0.0:8888: connect: connection refused"
What am I doing wrong?
Have a look at Drone services. It allows you to bring up a container as part of your pipeline and access its ports.
In your case you can bring up the Gogs container like this:
services:
- name: gogs
image: gogs/gogs
And then use it like this in your pipeline steps:
steps:
- name: test and build
image: golang
commands:
- curl "http://gogs"
-
...
(this assumes the gogs container listens on port 80. If it's a different port then you need to adjust the URI).
Hint: the name of the service is the DNS name of the container.
Related
I have a project, which consist of Go application + Nginx + Db(Postgres). All are building in docker containers.
It is my docker-compose.yml file:
version: "3"
services:
db:
image: postgres:10
environment:
- POSTGRES_PASSWORD=DatabasePassword
- POSTGRES_USER=egor
- POSTGRES_DB=postgres
expose:
- 5432
backend:
build: .
environment:
- POSTGRES_URL=postgres://egor:DatabasePassword#db:5432/postgres?sslmode=disable
- LISTEN_ADDRESS=:5432
depends_on:
- db
proxy:
image: nginx
volumes:
- type: bind
source: ./nginx.conf
target: /etc/nginx/nginx.conf
ports:
- 80:80
depends_on:
- backend
- db
it is my go application:
package main
import (
"database/sql"
"fmt"
"time"
_ "github.com/lib/pq"
"log"
"net/http"
"github.com/caarlos0/env"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type config struct {
PostgresUri string `env:"POSTGRES_URL" envDefault:"postgres://root:pass#localhost:5432/postgres?sslmode=disable"`
ListenAddress string `env:"LISTEN_ADDRESS" envDefault:":7000"`
//PostgresHost string `env:"POSTGRES_HOST" envDefault:":l"`
//PostgresUser string `env:"POSTGRES_USER" envDefault:":root"`
//PostgresPassword string `env:"POSTGRES_PASSWD" envDefault:":qwerty"`
//PostgresName string `env:"POSTGRES_NAME" envDefault:":postgres"`
}
var (
db *sql.DB
errorsCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gocalc_errors_count",
Help: "Gocalc Errors Count Per Type",
},
[]string{"type"},
)
requestsCount = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "gocalc_requests_count",
Help: "Gocalc Requests Count",
})
)
func main() {
var err error
// Initing prometheus
prometheus.MustRegister(errorsCount)
prometheus.MustRegister(requestsCount)
// Getting env
cfg := config{}
if err = env.Parse(&cfg); err != nil {
fmt.Printf("%+v\n", err)
}
time.Sleep(time.Second)
fmt.Println("Sleep over!")
// Connecting to database
//psqlInfo := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=5432 sslmode=disable",
// cfg.PostgresHost,cfg.ListenAddress,cfg.PostgresUser,cfg.PostgresPassword,cfg.PostgresName)
//db, err := sql.Open("postgres", "host=db user=egor password=DatabasePassword dbname=postgres port=5432 sslmode=disable")
db, err = sql.Open("postgres",cfg.PostgresUri)
if err != nil {
log.Fatalf("Can't connect to postgresql: %v", err)
}
defer db.Close()
err = db.Ping()
if err != nil {
log.Fatalf("Can't ping database: %v", err)
}
http.HandleFunc("/", handler)
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(cfg.ListenAddress, nil))
}
func handler(w http.ResponseWriter, r *http.Request) {
requestsCount.Inc()
keys, ok := r.URL.Query()["q"]
if !ok || len(keys[0]) < 1 {
errorsCount.WithLabelValues("missing").Inc()
log.Println("Url Param 'q' is missing")
http.Error(w, "Bad Request", 400)
return
}
q := keys[0]
log.Println("Got query: ", q)
var result string
sqlStatement := fmt.Sprintf("SELECT (%s)::numeric", q)
row := db.QueryRow(sqlStatement)
err := row.Scan(&result)
if err != nil {
log.Println("Error from db: %s", err)
errorsCount.WithLabelValues("db").Inc()
http.Error(w, "Internal Server Error", 500)
return
}
fmt.Fprintf(w, "query %s; result %s", q, result)
}
And my nginx configuration:
events{
worker_connections 1024;
}
http{
server {
listen 80;
server_name localhost;
location / {
proxy_pass http://backend:7000;
}
}
}
But when i'm going to try page in browser, i see error page - 502 Bad Gateway nginx.
It is my log:
2022/11/08 23:41:24 [error] 29#29: *1 connect() failed (111: Connection refused) while connecting to upstream, client: xxx.xx.x.x, server: localhost, request: "GET / HTTP/1.1", upstream: "http://xxx.xx.x.x:7000/", host: "0.0.0.0"
What is problem? All services work correctly, only nginx reversy proxy has error
I just put together a small project that represents your scenario. This is the repository structure:
webapp/
nginx/
Dockerfile
nginx.conf
web/
Dockerfile
main.go
docker-compose.yaml
The content of each file are as follows.
nginx/nginx.conf
events{}
http {
server {
listen 80;
location / {
proxy_pass http://backend:7000;
}
}
}
More or less is your same file.
nginx/Dockerfile
FROM nginx
EXPOSE 80
COPY nginx.conf /etc/nginx/nginx.conf
Here, we specify instructions to build the nginx container. We expose only the port 80.
web/main.go
package main
import (
"fmt"
"net/http"
)
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Hello, World!\n")
})
http.ListenAndServe(":7000", nil)
}
Simple HTTP server with a hard-coded reply. This HTTP server listens for requests on port 7000.
web/Dockerfile
FROM golang:1.12.7-alpine3.10 AS build
WORKDIR /go/src/app
COPY ./main.go ./main.go
RUN go build -o ./bin/gowebserver ./main.go
FROM alpine:latest
COPY --from=build /go/src/app/bin /go/bin
EXPOSE 7000
ENTRYPOINT go/bin/gowebserver
Here, we use the multi-stage build. In the first section we build the HTTP server while in the second one, we copy the executable on a leaner base image of Docker. We expose port 7000 of this container.
docker-compose.yaml
version: "3"
services:
backend:
build: "./web"
expose:
- "7000"
nginx:
build: "./nginx"
ports:
- "80:80"
depends_on:
- "backend"
Here, is the last part that connects all. We expose to the outside only the port 80. Internally, the backend service exposes port 7000 to be contacted by the nginx service.
To spin up everything, you've to run these two commands (in the root folder of the project):
docker-compose build
docker-compose up
To test this solution you've to use your internal IP address (in my case was something like 192.168.1.193) and navigate to the URL http://192.168.1.193/ which should give you an Hello, World! message.
Let me know if this solves your issue!
I'm new to grpc_web and envoy.
Please help me to setup following things,
GRPC_Go server is running on ec2 instance as a docker container
Dart web client is running on local pc
Need to make grpc call request from dart web app to grpc_go server
Used envoy proxy for the request forward. Envoy proxy is running as a container in same ec2 instance
I'm getting the following error "Response: null, trailers: {access-control-allow-credentials: true, access-control-allow-origin: http://127.0.0.1:9000, vary: Origin})".
Grpc_Go:
package main
import (
"context"
"flag"
"fmt"
"log"
"net"
"google.golang.org/grpc"
pb "google.golang.org/grpc/examples/helloworld/helloworld"
)
var (
port = flag.Int("port", 50051, "The server port")
)
// server is used to implement helloworld.GreeterServer.
type server struct {
pb.UnimplementedGreeterServer
}
// SayHello implements helloworld.GreeterServer
func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
log.Printf("Received: %v", in.GetName())
return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil
}
func (s *server) SayHelloAgain(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply,
error)
{
return &pb.HelloReply{Message: "Hello again " + in.GetName()}, nil
}
func main() {
flag.Parse()
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
pb.RegisterGreeterServer(s, &server{})
log.Printf("server listening at %v", lis.Addr())
if err := s.Serve(lis); err != nil {
log.Fatalf("failed to serve: %v", err)
}
}
GRPC_dart_client:
import 'package:grpc/grpc_web.dart';
import 'package:grpc_web/app.dart';
import 'package:grpc_web/src/generated/echo.pbgrpc.dart';
void main() {
final channel = GrpcWebClientChannel.xhr(Uri.parse('http://ec2-ip:8080'));
final service = EchoServiceClient(channel);
final app = EchoApp(service);
final button = querySelector('#send') as ButtonElement;
button.onClick.listen((e) async {
final msg = querySelector('#msg') as TextInputElement;
final value = msg.value!.trim();
msg.value = '';
if (value.isEmpty) return;
if (value.indexOf(' ') > 0) {
final countStr = value.substring(0, value.indexOf(' '));
final count = int.tryParse(countStr);
if (count != null) {
app.repeatEcho(value.substring(value.indexOf(' ') + 1), count);
} else {
app.echo(value);
}
} else {
app.echo(value);
}
});
}
envoy.yaml:
access_log_path: /tmp/admin_access.log
address:
socket_address: { address: 0.0.0.0, port_value: 9901 }
static_resources:
listeners:
- name: listener_0
address:
socket_address: { address: 0.0.0.0, port_value: 8080 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"#type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
codec_type: auto
stat_prefix: ingress_http
route_config:
name: local_route
virtual_hosts:
- name: local_service
domains: ["*"]
routes:
- match: { prefix: "/" }
route:
cluster: echo_service
timeout: 0s
max_stream_duration:
grpc_timeout_header_max: 0s
cors:
allow_origin_string_match:
- prefix: "*"
allow_methods: GET, PUT, DELETE, POST, OPTIONS
allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout
max_age: "1728000"
expose_headers: custom-header-1,grpc-status,grpc-message
http_filters:
- name: envoy.filters.http.grpc_web
- name: envoy.filters.http.cors
- name: envoy.filters.http.router
clusters:
- name: echo_service
connect_timeout: 0.25s
type: logical_dns
http2_protocol_options: {}
lb_policy: round_robin
load_assignment:
cluster_name: cluster_0
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app
port_value: 50051
Grpc_go_docker_file:
# Install git.
# Git is required for fetching the dependencies.
RUN apk update && apk add --no-cache git
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main .
# Start a new stage from scratch
FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/
# Copy the Pre-built binary file from the previous stage. Observe we also copied the .env file
COPY --from=builder /app/main .
# Expose port 50051 to the outside world
EXPOSE 50051
CMD ["./main"]
Envoy_Docker:
COPY envoy.yaml /etc/envoy/envoy.yaml
CMD /usr/local/bin/envoy -c /etc/envoy/envoy.yaml -l trace --log-path /tmp/envoy_info.log
I'm stuck with it more than two days, please help me. Thanks in advance
Thank you all, for your reply.
I fixed this issue with IP of the ec2 instance.
clusters:
- name: echo_service
connect_timeout: 0.25s
type: logical_dns
http2_protocol_options: {}
lb_policy: round_robin
load_assignment:
cluster_name: cluster_0
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app
port_value: 50051
Instead of container 'address: app' (app is container name) in the envoy.yaml, I used the ip of ec2 instance and container port now envoy is forwarding the request to server.
I am trying to run docker from golang and when I tried the code mentioned in the docker official
site, am getting these error. wondering if I have an incorrect vendor
resp, err := cli.ContainerCreate(ctx,
&container.Config{
Image: imageName,
},
nil,
nil,
"")
not enough arguments in call to cli.ContainerCreate
have (context.Context, *container.Config, nil, nil, string)
want (context.Context, *container.Config, *container.HostConfig, *network.NetworkingConfig, *v1.Platform, string)
In this example, you can see what *v1.Platform can be initialized to:
resp, err := cli.ContainerCreate(ctx, &container.Config{Hostname: "my-rabbit",
Image: "rabbitmq:3.7.8-management",
Tty: true,
}, &container.HostConfig{RestartPolicy: container.RestartPolicy{Name: "always"}, PortBindings: bindings}, &network.NetworkingConfig{}, "rabbit")
if err != nil {
panic(err)
}
The &network.NetworkingConfig{} references github.com/docker/docker/api/types/network#NetworkingConfig
Carries the networking configs specified in the docker run and docker network connect commands
I am trying to create a container for testing purpose with https://pkg.go.dev/github.com/fsouza/go-dockerclient?tab=doc as follows:
client, err := docker.NewClientFromEnv()
NoError(t, err, err)
ctx := context.Background()
gogs, err := client.CreateContainer(docker.CreateContainerOptions{
Name: "gogs",
Config: &docker.Config{
Image: "gogs/gogs:0.11.91",
},
HostConfig: &docker.HostConfig{
PublishAllPorts: true,
AutoRemove: true,
Binds: []string{dest, "/data"},
PortBindings: map[docker.Port][]docker.PortBinding{
"3000/tcp": {{HostIP: "0.0.0.0", HostPort: "8888"}}},
},
Context: ctx,
})
NoError(t, err, err)
err = client.StartContainer(gogs.ID, nil)
NoError(t, err, err)
The problem is, the bind does not work. What am I trying to achieve is:
docker run --rm -it -p 8888:3000 --name=gogs --mount type=bind,source=`pwd`/gogs/data,target=/data gogs/gogs
What am I doing wrong?
When mounting a path you can attach Mounts to the HostConfig like this:
&container.HostConfig{
Mounts: []mount.Mount{
{
Type: mount.TypeBind,
Source: "/source",
Target: "/target",
},
},
}
This will mount /source in to the container.
I am trying to create a containerSource for knative service. When I use docker run for the image it gives the output ("or the error from the code"). However when I apply the yaml file then kubectl log shows 'standard_init_linux.go:211: exec user process caused "no such file or directory"'. docker run shows that it is able to find the exec file. So I am not able to understand whats wrong. Someone please guide me through.
my yaml file:
apiVersion: sources.eventing.knative.dev/v1alpha1
kind: ContainerSource
metadata:
labels:
controller-tools.k8s.io: "1.0"
name: cloudevents-source
spec:
image: docker.io/username/pkt-event:latest
args:
- '--debug=true'
sink:
apiVersion: serving.knative.dev/v1alpha1
kind: Service
name: event-display
my go code for the dockerimage is:
package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"time"
"github.com/satori/go.uuid"
"knative.dev/eventing-contrib/pkg/kncloudevents"
"encoding/json"
// "io/ioutil"
// "knative.dev/eventing-contrib/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents"
"github.com/cloudevents/sdk-go/pkg/cloudevents"
"github.com/cloudevents/sdk-go/pkg/cloudevents/types"
"github.com/kelseyhightower/envconfig"
)
var (
eventSource string
eventType string
sink string
)
//var u, _ = uuid.NewV4()
var debug = flag.Bool("debug", false, "Enable debug mode (print more information)")
var source = flag.String("source", uuid.NewV4().String(), "Set custom Source for the driver")
func init() {
flag.StringVar(&eventSource, "eventSource", "", "the event-source (CloudEvents)")
flag.StringVar(&eventType, "eventType", "dev.knative.eventing.samples.pkt", "the event-type (CloudEvents)")
flag.StringVar(&sink, "sink", "", "the host url to send pkts to")
}
type envConfig struct {
// Sink URL where to send heartbeat cloudevents
Sink string `envconfig:"SINK"`
}
func main() {
flag.Parse()
var env envConfig
if err := envconfig.Process("", &env); err != nil {
log.Printf("[ERROR] Failed to process env var: %s", err)
os.Exit(1)
}
if env.Sink != "" {
sink = env.Sink
}
if eventSource == "" {
eventSource = fmt.Sprintf("https://knative.dev/eventing-contrib/cmd/heartbeats/#local/demo")
log.Printf("Source: %s", eventSource)
}
client, err := kncloudevents.NewDefaultClient(sink)
if err != nil {
log.Fatalf("failed to create client: %s", err.Error())
}
var period time.Duration
period = time.Duration(1) * time.Second
ticker := time.NewTicker(period)
for {
content := "Send data"
data, err := json.Marshal(content)
if err != nil {
fmt.Println(err)
}
event := cloudevents.Event{
Context: cloudevents.EventContextV02{
Type: "packet.invoke",
Source: *types.ParseURLRef(eventSource),
/*Extensions: map[string]interface{}{
"the": 42,
"heart": "yes",
"beats": true,
},*/
}.AsV02(),
Data: data,
}
if *debug{
log.Printf("Sending event %v", event)
} else {
if _, err := client.Send(context.TODO(), event); err != nil {
log.Printf("failed to send cloudevent: %s", err.Error())
}
}
<-ticker.C
}
}
And Dockerfile is:
FROM golang:1.12 as builder
RUN go version
WORKDIR ${GOPATH}/src/Event-driver
COPY ./ ./
RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
##RUN dep ensure
RUN dep init
RUN dep ensure
RUN CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -v -o my-event my-event.go
RUN pwd && ls
FROM scratch
#FROM ubuntu:disco
COPY --from=builder /go/src/Event-driver/my-event /
ENTRYPOINT ["/my-event"]
That problem occurs because you're trying to run your binary from bash, but scratch has no bash.
I'm normally using alpina instead. To build for alpina you need the same environment variables, so probably you only need to change a second stage image.