I'm developing a golang web project with Redigo to connect to a redis docker container.
While the golang web application is in running state, I've changed the value of a redis key using SET MyFlag true in redis-cli. But this is not reflecting in my webapp. When MyFlag is fetched using flag, err := redis.Bool(conn.Do("GET", "MyFlag")), it gives the old value.
But, after the webapp is restarted, the same command fetches the new value.
conn is retrieved from a redis connection pool. This is the configuration for redis pool
redisPool = &redis.Pool{
MaxIdle: 5,
IdleTimeout: 240 * time.Second,
MaxActive: 10,
Wait: true,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", db.GetUrl())
if err != nil {
return nil, err
}
password := db.GetPassword()
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
_ = c.Close()
return nil, err
}
}
return c, nil
},
}
Is this the issue of caching in Redis/Redigo/Docker ?
Related
We are building multiple docker images in the cloud. This is done using this code -
func BuildM1Image(tags []string, dockerfile string, contextPath string, dockerRepoUrl string) error {
dockerExecutable, _ := exec.LookPath("docker")
awsExecutable, _ := exec.LookPath("aws")
toTag := tags[0]
Log("building image with tag " + toTag)
// to push --> I need the AWS credentials to live in the cloud, pull them
os.Setenv("AWS_ACCESS_KEY_ID", Secrets[ECR_ACCESS_KEY_NAME])
os.Setenv("AWS_SECRET_ACCESS_KEY", Secrets[ECR_SECRET_ACCESS_KEY_NAME])
ecrGetCredentialsCMD := &exec.Cmd{
Path: awsExecutable,
Args: []string{awsExecutable, "ecr", "get-login-password", "--region", GetRegion()},
// Stderr: os.Stderr,
// Stdout: os.Stdout,
}
out, _ := ecrGetCredentialsCMD.CombinedOutput()
// if err != nil {
// errorChannel <- err
// return
// }
dockerEcrLoginCMD := &exec.Cmd{
Path: dockerExecutable,
Args: []string{dockerExecutable, "login", "--username", "AWS", "-p", string(out), dockerRepoUrl},
}
if err := dockerEcrLoginCMD.Run(); err != nil {
fmt.Println("Docker login failed")
fmt.Println("error: ", err)
return err
}
buildDockerImage := &exec.Cmd{
Path: dockerExecutable,
Args: []string{dockerExecutable, "buildx", "build", "--platform", "linux/amd64", "-t", toTag, "-f", dockerfile, contextPath},
}
if err := buildDockerImage.Run(); err != nil {
fmt.Println("docker build failed")
logError(err)
return err
}
return nil
}
I put the this function inside of a goroutine and kicked off multiple builds. However, the time taken on the builds is slower than if I were to do this sequentially. This only happens in an EC2 instance and not locally (locally we're running this with a M1 mac and it's working as expected).
Why would this happen?
On the EC2 instance we've tried -
Increasing the compute/storage
Increasing the IOPS
Thanks!
I am using go-redis to connect to Redis server running on docker desktop while running my go app straight on my mac.
This my client setup:
package redis
import (
"fmt"
"os"
"github.com/go-redis/redis/v8"
)
var redisClient *RedisClient
type RedisClient struct {
*redis.Client
}
func GetRedisClient() *RedisClient {
if redisClient != nil {
return redisClient
}
host := os.Getenv("REDIS_HOST")
port := os.Getenv("REDIS_PORT")
password := os.Getenv("REDIS_PASS")
client := redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%s", host, port),
Password: password, // no password set
DB: 0,
})
redisClient = &RedisClient{
Client: client,
}
return redisClient
}
Docker:
version: "3.8"
services:
redis:
container_name: redis
image: redis:6.2
ports:
- "6379:6379"
ulimits:
nofile:
soft: 65536
hard: 65536
The app will expose websocket connections to drivers that will communicate their current location every second and then save them in Redis using GEOADD.
Also the app will expose another set of websocket connections to the same drivers for general notifications if any using BRPOP.
After 70 driver of websocket connections I get errors from the extra drivers trying to connect. The errors come from the function that saves the location to Redis. The errors I get:
dial tcp [::1]:6379: socket: too many open files
and sometimes dial tcp: lookup localhost: no such host
func (r *RedisClient) SetPoint(ctx context.Context, item *Identifier, loc *Location) error {
geoLocaiton := &redis.GeoLocation{Name: item.Id, Latitude: loc.Lat, Longitude: loc.Lng}
if err := r.GeoAdd(ctx, item.key(), geoLocaiton).Err(); err != nil {
fmt.Println("error adding geo", err)
return errors.New("failed to set point")
}
return nil
}
For general notifications (timeout on the pulling is zero) meaning infinate:
type DriverData struct {
Status OrderStatusType `json:"status,omitempty"`
DriverId uint `json:"driver_id,omitempty"`
UserId uint `json:"user_id,omitempty"`
}
func (config *Config) DriverOrderStatus(c *gin.Context) {
driverID := utils.ToUint(auth.GetToken(c).Subject)
ctx := c.Request.Context()
// order := models.GetOrder(config.Db)
// var _ = order.GetActiveOrderForUser(driverID)
wsconn, err := websocket.Accept(c.Writer, c.Request, &websocket.AcceptOptions{InsecureSkipVerify: true})
if err != nil {
return
}
// if order.ID != 0 {
// var _ = wsjson.Write(ctx, wsconn, &UserData{Order: order, Status: order.Status, Driver: order.Driver})
// } else {
// var _ = wsjson.Write(ctx, wsconn, &UserData{ResetOrder: true})
// }
defer wsconn.Close(websocket.StatusInternalError, "")
closeRead := wsconn.CloseRead(ctx)
driverDataCh := make(chan *DriverData, 1000)
go func() {
loop:
for {
select {
case <-closeRead.Done():
break loop
default:
if status, err := config.Redis.DriverPullStatus(ctx, driverID); err == nil {
driverDataCh <- &DriverData{Status: status.Status, DriverId: status.DriverID, UserId: status.UserID}
}
}
}
fmt.Println("redis pulling data is over")
}()
loop:
for {
select {
case <-closeRead.Done():
break loop
case driverData := <-driverDataCh:
if err := wsjson.Write(ctx, wsconn, driverData); err != nil {
break loop
}
}
}
fmt.Println("sending updates to user is over")
}
This is Redis server info:
127.0.0.1:6379> info
# Server
redis_version:6.2.6
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:a0adc3471b8cfa72
redis_mode:standalone
os:Linux 5.10.47-linuxkit x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:10.3.1
process_id:1
process_supervised:no
run_id:d92005e2ccb89ea8e3be57e3bb1b79e0e323c2a7
tcp_port:6379
server_time_usec:1654937072463352
uptime_in_seconds:325287
uptime_in_days:3
hz:10
configured_hz:10
lru_clock:10769904
executable:/data/redis-server
config_file:
io_threads_active:0
# Clients
connected_clients:104
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:48
client_recent_max_output_buffer:0
blocked_clients:81
tracking_clients:0
clients_in_timeout_table:0
# Memory
used_memory:3081168
used_memory_human:2.94M
used_memory_rss:5791744
used_memory_rss_human:5.52M
used_memory_peak:5895528
used_memory_peak_human:5.62M
used_memory_peak_perc:52.26%
used_memory_overhead:2944804
used_memory_startup:809880
used_memory_dataset:136364
used_memory_dataset_perc:6.00%
allocator_allocated:3166992
allocator_active:3862528
allocator_resident:6742016
total_system_memory:4125036544
total_system_memory_human:3.84G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.22
allocator_frag_bytes:695536
allocator_rss_ratio:1.75
allocator_rss_bytes:2879488
rss_overhead_ratio:0.86
rss_overhead_bytes:-950272
mem_fragmentation_ratio:1.88
mem_fragmentation_bytes:2712392
mem_not_counted_for_evict:0
mem_replication_backlog:0
mem_clients_slaves:0
mem_clients_normal:2134588
mem_aof_buffer:0
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0
# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:3636
rdb_bgsave_in_progress:0
rdb_last_save_time:1654936992
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:450560
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0
# Stats
total_connections_received:1271
total_commands_processed:296750
instantaneous_ops_per_sec:45
total_net_input_bytes:27751095
total_net_output_bytes:1254190
instantaneous_input_kbps:4.16
instantaneous_output_kbps:12.46
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:18136
evicted_keys:0
keyspace_hits:10
keyspace_misses:3567
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:6453
total_forks:41
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:6
dump_payload_sanitizations:0
total_reads_processed:297924
total_writes_processed:295658
io_threaded_reads_processed:0
io_threaded_writes_processed:0
# Replication
role:master
connected_slaves:0
master_failover_state:no-failover
master_replid:fc426cf72670e6ad09221bcb9c3423a1e1fab47e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
# CPU
used_cpu_sys:428.939381
used_cpu_user:123.850311
used_cpu_sys_children:0.755309
used_cpu_user_children:0.065924
used_cpu_sys_main_thread:428.485425
used_cpu_user_main_thread:123.679341
# Modules
# Errorstats
errorstat_ERR:count=6
# Cluster
cluster_enabled:0
# Keyspace
db0:keys=6,expires=0,avg_ttl=0
After a lot of searching, it turns out that is caused by the max "open file descriptor". Every websocket connection will open a file descriptor. Every machine has a limit. In linux/unix, this is defined under ulimit.
More into that in this article.
In order to update ulimit in mac, refer to this post.
I am trying to access rabbitmq cluster where TLS is enabled. I have written sample go app which trying to connect to the rabbitmq server using set of client certificates and client keys.
I am facing error -
Error is - Exception (403) Reason: "username or password not allowed"
panic: Exception (403) Reason: "username or password not allowed"
My code snippet
package main
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"log"
"github.com/streadway/amqp"
)
func main() {
fmt.Println("Go RabbitMQ Consumer Tutorial")
fmt.Println("Testing ClusterIP service connection over TLS")
cert, err := tls.LoadX509KeyPair("client.crt", "client.key")
if err != nil {
panic(err)
}
//Load CA cert.
caCert, err := ioutil.ReadFile("ca.crt") // The same you configured in your MQ server
if err != nil {
log.Fatal(err)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
TlsConfig1 := &tls.Config{
Certificates: []tls.Certificate{cert}, // from tls.LoadX509KeyPair
RootCAs: caCertPool,
InsecureSkipVerify: true,
// ...other options are just the same as yours
}
conn, err := amqp.DialTLS("amqps://<username>:<password>#<rabbitmq-service-name>.<namespace>.svc.cluster.local:5671/", TlsConfig1)
if err != nil {
fmt.Println("Error is -", err)
panic(err)
}
fmt.Println("Connected to consumer successfully")
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
fmt.Println(err)
}
defer ch.Close()
if err != nil {
fmt.Println(err)
}
msgs, err := ch.Consume(
"TestQueue",
"",
true,
false,
false,
false,
nil,
)
forever := make(chan bool)
go func() {
for d := range msgs {
fmt.Printf("Recieved Message: %s\n", d.Body)
}
}()
fmt.Println("Successfully Connected to our RabbitMQ Instance")
fmt.Println(" [*] - Waiting for messages")
<-forever
}
The code snippet is running as pod in the EKS cluster where my rabbitmq cluster is running.
After going through the rabbitmq TLS debugging doc- https://www.rabbitmq.com/access-control.html I figured out that I was giving wrong username and password.
NOTE: if any one who is also getting the same erorr - please confirm that you are using correct username and password.
you can check the username and password for the cluster by logging into the rabbitmq cluster pod.
This happens to me on version 11.6.0 but not on 10.3.9.
I am currently working on a server that is designed to be run in a Docker container.
Here is my setup method for my tests:
func TestMain(m *testing.M) {
schedulerName := "scheduler1"
IP, err := container.StartNewScheduler(schedulerName)
if err != nil {
log.Println("Could not create container.")
log.Fatal(err)
}
serverIP = IP
code := m.Run()
cleanupContainer(schedulerName)
os.Exit(code)
}
The line container.StartNewScheduler(schedulername) boots up a new docker container called "scheduler1" and tells it to run the server inside of it.
Next I run my tests with the container running in the background, right now I only have one test.
func TestNewScheduler(t *testing.T) {
testCodeInput := "THIS IS A TEST"
requestBody, err := json.Marshal(map[string]string{
"Code": fmt.Sprintf("print(\"%s\")", testCodeInput),
})
if err != nil {
t.Fatal(err)
}
url := fmt.Sprintf("http://%s:%d/execute/python", serverIP, 3000)
contentType := "application/json"
body := bytes.NewBuffer(requestBody)
response := post(url, contentType, body, t)
actual := parseOutput(response.Body, t)
response.Body.Close()
expected := fmt.Sprintf("{\"Stdout\":\"%s\\n\"}", testCodeInput)
if actual != expected {
t.Fatalf("Expected %s, but got %s", expected, actual)
}
}
The problem that I am running into is sometimes I get a connection refused and sometimes I don't.
server_container_test.go:51: Post http://172.20.0.2:3000/execute/python: dial tcp 172.20.0.2:3000: connect: connection refused
I noticed that whenever I try and debug the issue everything seems to work fine. My running theory is because when I step through my code the container has more time to start up and get the server running inside it.
In order to test my Hypothesis I added a second post call in my post method with a timer set before I call it.
func post(url string, contentType string, body io.Reader, t *testing.T) *http.Response {
t.Helper()
response, err := http.Post(url, contentType, body)
if err != nil {
//There is an error where the container takes a second to boot up and so
//the scheduler isn't running when the first request is sent, so we try
//one more time here and check again.
time.Sleep(100 * time.Millisecond) <--- Right here
response, err = http.Post(url, contentType, body)
if err != nil {
t.Fatal(err)
}
}
return response
}
Does anyone else have any guesses as to what could be causing me this issue?
If my hypothesis is correct is this the best way to fix this? Is it a bad idea to add a time.Sleep to your tests?
Thanks!
Ok so after some more thought I changed up my source code, please let me know if you think this is a good solution to my problem. I am still learning Go and HTTP servers so any input is appreciated.
Here is my fix/idea:
Previously once the container was created I just returned it's IP address and forgot about it.
Now I create a go routine that repeatedly tries to send a POST request to the server. If it doesn't fail then I send true through a channel and close the function.
IP := info.NetworkSettings.Networks[networkName].IPAddress
works := make(chan bool)
ctx, canelRoutine := context.WithCancel(context.Background())
defer canelRoutine()
go func(ctx context.Context) {
requestBody, _ := json.Marshal(map[string]string{
"Code": "print()",
})
select {
case <-ctx.Done():
return
default:
for {
_, err := http.Post(
fmt.Sprintf("http://%s:%d/execute/python", IP, 3000),
"application/json",
bytes.NewBuffer(requestBody),
)
if err == nil {
works <- true
return
}
}
}
}(ctx)
After sending the goroutine off I create a timer and and wait for either the timer to return or the goroutine.
timer := time.After(500 * time.Millisecond)
select {
case <-works:
return IP, nil
case <-timer:
return IP, &UnreachableContainerError{name: schedulerName}
}
The upside to this solution is I have now introduced an UnreachableContainerError which allows me to be more specific about my error messages and it can be checked on the receiving side. I also send the IP address back either way just in case the client needs it for some other reason.
Here is the full StartNewScheduler method in case you wanted to see it.
//StartNewScheduler starts a new scheduler with the given options.
//returns the IP address for the given scheduler.
func StartNewScheduler(schedulerName string) (string, error) {
///Defaults
dockerfile := "Dockerfile_standard"
networkName := "scheduler-cluster"
imageID := "lkelly93/scheduler_image:latest"
cli, err := client.NewEnvClient()
if err != nil {
return "", err
}
err = createDefaultImageIfNeeded(
cli,
imageID,
dockerfile)
if err != nil {
return "", err
}
err = createSchedulerClusterNetworkIfNeeded(cli, networkName)
if err != nil {
return "", err
}
ctx := context.Background()
resp, err := cli.ContainerCreate(
ctx,
&container.Config{Image: imageID},
&container.HostConfig{
NetworkMode: container.NetworkMode(networkName),
Privileged: true,
},
nil,
schedulerName,
)
if err != nil {
return "", err
}
err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
if err != nil {
return "", err
}
//Get container IP
info, err := cli.ContainerInspect(ctx, resp.ID)
if err != nil {
return "", err
}
IP := info.NetworkSettings.Networks[networkName].IPAddress
works := make(chan bool)
ctx, canelRoutine := context.WithCancel(context.Background())
defer canelRoutine()
go func(ctx context.Context) {
requestBody, _ := json.Marshal(map[string]string{
"Code": "print()",
})
select {
case <-ctx.Done():
return
default:
for {
_, err := http.Post(
fmt.Sprintf("http://%s:%d/execute/python", IP, 3000),
"application/json",
bytes.NewBuffer(requestBody),
)
if err == nil {
works <- true
return
}
}
}
}(ctx)
timer := time.After(500 * time.Millisecond)
select {
case <-works:
return IP, nil
case <-timer:
return IP, &UnreachableContainerError{name: schedulerName}
}
}
You could run the test code inside of a container, started via docker compose with the option depends_on.
I am trying to execute a command (let's say "pwd) using Docker Go lang SDK, and I expect it returns the working directory on the container. But it returns nothing back. I am not sure what is the issue.
rst, err := cli.ContainerExecCreate(context.Background(), "0df7c1d9d185b1da627efb983886a12fefc32120d035b34e97c3ad13da6dd9cc", types.ExecConfig{Cmd: []string{"pwd"}})
if err != nil {
panic(err)
}
//res, err := cli.ContainerExecInspect(context.Background(), rst.ID)
//print(res.ExitCode)
response, err := cli.ContainerExecAttach(context.Background(), rst.ID, types.ExecStartCheck{})
if err != nil {
panic(err)
}
defer response.Close()
data, _ := ioutil.ReadAll(response.Reader)
fmt.Println(string(data))
GOROOT=/usr/local/Cellar/go/1.13.5/libexec #gosetup
GOPATH=/Users/pt/go #gosetup
/usr/local/Cellar/go/1.13.5/libexec/bin/go build -o /private/var/folders/yp/hh3_03d541x0r6t7_zwqqhqr0000gn/T/___go_build_main_go /Users/pt/go/src/awesomeProject/main.go #gosetup
/private/var/folders/yp/hh3_03d541x0r6t7_zwqqhqr0000gn/T/___go_build_main_go #gosetup
### It does not print the working directory ###
Process finished with exit code 0
This is solved by the following config:
optionsCreate := types.ExecConfig{
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"ls", "-a"},
}