nginx connection refused - tcp connection error - docker

I have following nginx config:
# https://github.com/KyleAMathews/docker-nginx/blob/master/nginx.conf
# https://linode.com/docs/web-servers/nginx/configure-nginx-for-optimized-performance/
# https://docs.gunicorn.org/en/stable/deploy.html
worker_processes 1;
events {
worker_connections 2000; # increase if you have lots of clients
accept_mutex off; # set to 'on' if nginx worker_processes > 1
use epoll; # Enable epoll for Linux 2.6+
# 'use kqueue;' to enable for FreeBSD, OSX
}
http {
include mime.types;
# fallback in case we can't determine a type
default_type application/octet-stream;
sendfile on;
upstream app_server {
# ip_hash; # For load-balancing
#
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response
server unix:/nginx/gunicorn.socket fail_timeout=0;
# for a TCP configuration
# server web:8080 fail_timeout=0;
keepalive 32;
}
server {
access_log off;
listen 8080 deferred;
charset utf-8;
keepalive_timeout 75s;
# https://thoughts.t37.net/nginx-optimization-understanding-sendfile-tcp-nodelay-and-tcp-nopush-c55cdd276765
# tcp_nopush on;
# tcp_nodelay on;
gzip on;
gzip_min_length 1000;
gzip_comp_level 2;
# text/html is always included by default
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml;
gzip_disable "MSIE [1-6]\.";
location /static {
alias /nginx/staticfiles;
expires 365d;
}
location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://app_server/;
proxy_set_header X-Forwarded-Host $server_name;
proxy_set_header X-Real-IP $remote_addr;
add_header Front-End-Https on;
}
}
}
but when i try to hit http://localhost:8000/cgw/v1/chains/43214913/about/master-copies
I get this error:
{
"code": 1337,
"message": "reqwest::Error { kind: Request, url: Url { scheme: \"http\", host: Some(Ipv4(127.0.0.1)), port: Some(8080), path: \"/api/v1/about/master-copies/\", query: None, fragment: None }, source: hyper::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) }"
}
any idea what i'm doing wrong in nginx config ( totally newbie with nginx sorry about that )

Related

A Server restarted and Website giving 502 Bad Gateway

So i was asked to check this server out that after a restart is giving an 502 bad gateway message.
It's an nginx 1.10.3 in an ubuntu
I was looking around and what I can see is this:
in /var/log/nginx/error.log:
2020/03/18 02:29:45 [error] 3166#3166: *474 connect() to unix:/home/ubuntu/xxx/shared/tmp/sockets/puma.sock failed (111: Connection refused) while connecting to upstream, client: 190.162.83.37, server: 127.0.0.1, request: "GET / HTTP/1.1", upstream: "http://unix:/home/ubuntu/xxx/shared/tmp/sockets/puma.sock:/500.html", host: "domain.website.com"
I see this error multiple times. All I was told is that the website was working fine until it was rebooted. Apparently the server had been working non-stop for months.
The nginx.conf config is:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 1024;
# multi_accept on;
}
http {
uwsgi_connect_timeout 75s;
proxy_connect_timeout 600;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
fastcgi_send_timeout 600s;
fastcgi_read_timeout 600s;
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#mail {
# # See sample authentication script at:
# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
# # auth_http localhost/auth.php;
# # pop3_capabilities "TOP" "USER";
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
# server {
# listen localhost:110;
# protocol pop3;
# proxy on;
# }
#
# server {
# listen localhost:143;
# protocol imap;
# proxy on;
# }
#}
And the /etc/nginx/sites-enabled/default file:
upstream app {
# Path to Puma SOCK file, as defined previously
server unix:/home/ubuntu/xxx/shared/tmp/sockets/puma.sock fail_timeout=0;
}
server {
listen 80;
server_name 127.0.0.1 172.31.0.59;
root /home/ubuntu/xxx/current/public;
try_files $uri/index.html $uri #app;
location / {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Connection '';
proxy_pass http://app;
}
location ~ ^/(assets|fonts|system)/|favicon.ico|robots.txt|A8774B2003352D7DCA9406C3FADC610B.txt|sitemap.xml|video.mp4 {
gzip_static on;
expires max;
add_header Cache-Control public;
}
error_page 500 502 503 504 /500.html;
client_max_body_size 4G;
keepalive_timeout 10;
}
server {
listen 443 ssl;
server_name 127.0.0.1 172.31.0.59;
ssl_certificate /home/ubuntu/xxx/shared/cert/demain.website.com.crt;
ssl_certificate_key /home/ubuntu/xxx/shared/cert/demain.website.com.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
#maintenance
error_page 503 /maintenance.html;
location /maintenance.html {
# allow access to this specific page
}
#location / {
# return 503;
#}
# Tell Nginx and Passenger where your app's 'public' directory is
root /home/ubuntu/xxx/current/public;
try_files $uri/index.html $uri #app;
location / {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Connection '';
proxy_pass http://app;
}
location ~ ^/(assets|fonts|system)/|favicon.ico|robots.txt|A8774B2003352D7DCA9406C3FADC610B.txt|sitemap.xml|video.mp4 {
gzip_static on;
expires max;
add_header Cache-Control public;
}
error_page 500 502 503 504 /500.html;
client_max_body_size 4G;
keepalive_timeout 10;
# Turn on Passenger
#passenger_enabled on;
#passenger_ruby /usr/share/rvm/gems/ruby-2.4.2/wrappers/ruby;
#passenger_app_env stage;
}
I don't know what to do. Please Help.

Unable to connect docker login to nexus repo behind nginx

I have a computer trying to do:
docker login docker-repo.mydomain.com
docker-repo is handled by an nginx which has this config:
server {
listen 80;
server_name docker-repo.mydomain.com ;
# Redirect non-https traffic to https
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
server_name docker-repo.mydomain.com;
ssl_certificate /etc/nginx/ssl/docker-repo.mydomain.com.crt;
ssl_certificate_key /etc/nginx/ssl/docker-repo.mydomain.key;
server_tokens off;
ssl_dhparam /etc/nginx/ssl/dhparams.pem;
ssl_session_cache shared:SSL:10m;
ssl_protocols TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-CAMELLIA256-SHA:AES256-GCM-S$
gzip_proxied any;
gzip on;
gzip_min_length 1023;
gzip_types text/plain text/css text/js text/javascript text/xml application/json application/javascript application/x-javascript application/xml application/xml+rss;
proxy_send_timeout 610s;
proxy_read_timeout 610s;
proxy_max_temp_file_size 16384m;
proxy_redirect off;
proxy_buffers 32 4k;
send_timeout 610s;
client_max_body_size 0;
client_body_buffer_size 128k;
location / {
proxy_pass http://nexus.mydomain.com:8102;
proxy_set_header X-Custom-Referrer "https://docker-repo.mydomain.com:443";
proxy_set_header Host $http_host;
proxy_set_header Client-IP $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
And on nexus.mydomain.com i have sonatype nexus running a docker group with Http Proxy set to run on port 8102
Errors i'm getting:
$ docker login docker-repo.mydomain.com
Username: XXXX
Password:
Error response from daemon: Get https://docker-repo.mydomain.com/v2/: Service Unavailable
If i try to access the page https://docker-repo.mydomain.com from a browser I get the page: HTTP Error 400, Not a docker request
I i try to ping both docker-repo and nexus.mydomain.com it works fine.
Turns out that this was due to a proxy i was behind. Once i added the proxy config with the proper exclusions it worked fine.

can't connect rails puma server with nginx reverse proxy

Hello I am trying to setup a reverse proxy with nginx and docker container rails app, public static files are served correctly but can not access to my app. the nginx error log says:
2018/12/08 16:46:45 [error] 4093#4093: *350 could not find named location "#puma", client: xx.xxx.xxx.xx, server: my.app, request: "GET /en/users/sign_in HTTP/2.0",host: "my.app", referrer: "https://my.app/"
my nginx config is this:
upstream puma {
server 0.0.0.0:3000;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name my.app;
root /var/www/myapp/public;
# SSL
ssl_certificate /etc/letsencrypt/live/my.app/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/my.app/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/my.app/fullchain.pem;
include snippets/letsencrypt.conf;
include snippets/ssl.conf;
# reverse proxy
location / {
proxy_pass http://puma; # => http://0.0.0.0:3000
proxy_set_header Host $http_host; # => $host
proxy_set_header X-Forwarded-Proto $scheme; # => "https"
proxy_set_header X-Forwarded-Host $host; # => 0.0.0.0
proxy_set_header X-Forwarded-Port $server_port; # => 3000
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_cache_bypass $http_upgrade;
# index fallback
try_files $uri $uri/ /index.html;
}
# . files
location ~ /\. {
deny all;
}
# assets, media
location ~* \.(?:css(\.map)?|js(\.map)?|jpe?g|png|gif|ico|cur|heic|webp|tiff?)$ {
expires 7d;
access_log off;
}
# svg, fonts
location ~* \.(?:svgz?|ttf|ttc|otf|eot|woff2?)$ {
add_header Access-Control-Allow-Origin "*";
expires 7d;
access_log off;
}
# gzip
gzip on;
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_types text/plain text/css text/xml application/json application/javascript application/xml+rss application/atom+xml image/svg+xml;
# error pages
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /var/www/myapp/public;
}
error_page 404 /404.html;
location = /404.html {
root /var/www/myapp/public;
}
}
# subdomains redirect
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name _ *.my.app;
# SSL
ssl_certificate /etc/letsencrypt/live/my.app/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/my.app/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/my.app/chain.pem;
include snippets/letsencrypt.conf;
include snippets/ssl.conf;
return 301 https://my.app$request_uri;
}
# HTTP redirect
server {
listen 80;
listen [::]:80;
server_name _ .my.app my.app;
include snippets/letsencrypt.conf;
return 301 https://my.app$request_uri;
}
the rails container start thru docker-compose with this setup:
version: '3.2'
services:
web:
command: rails server -p '3000' -b '0.0.0.0' -e production
ports:
- '3000:3000'
I have tried also with unix socket to connect puma and nginx without success

Puma shows al requests coming from 127.0.0.1 when behind nginx

I am having a problem where the only IP Address that shows up in my rails log is 127.0.0.1, it appears that the remote ip is not getting proxy passed. I am unsure of what I a missing. Nginx is custom compiled within an omnibus package. and I have that build script below as well. If anyone can give me some insight that would be greatly appreciated.
Nginx Build Recipe:
name "nginx"
default_version "1.9.10"
dependency "pcre"
dependency "openssl"
source url: "http://nginx.org/download/nginx-#{version}.tar.gz",
md5: "64cc970988356a5e0fc4fcd1ab84fe57"
relative_path "nginx-#{version}"
build do
command ["./configure",
"--prefix=#{install_dir}/embedded",
"--with-http_ssl_module",
"--with-http_stub_status_module",
"--with-http_gzip_static_module",
"--with-http_v2_module",
"--with-http_realip_module",
"--with-ipv6",
"--with-debug",
"--with-ld-opt=-L#{install_dir}/embedded/lib",
"--with-cc-opt=\"-L#{install_dir}/embedded/lib -I#{install_dir}/embedded/include\""].join(" ")
command "make -j #{workers}", :env => {"LD_RUN_PATH" => "#{install_dir}/embedded/lib"}
command "make install"
end
Nginx Config:
user smart-mobile smart-mobile;
worker_processes 1;
error_log stderr;
pid nginx.pid;
daemon off;
events {
worker_connections 10240;
}
http {
#log_format combined '$remote_addr - $remote_user [$time_local] '
# '"$request" $status $body_bytes_sent '
# '"$http_referer" "$http_user_agent"';
#
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
gzip on;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_proxied any;
gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json;
proxy_cache_path proxy_cache keys_zone=smart-mobile:10m max_size=1g levels=1:2;
proxy_cache smart-mobile;
include /opt/smart-mobile/embedded/conf/mime.types;
include /var/opt/smart-mobile/nginx/conf/smart-mobile.conf;
}
Nginx Site Config:
upstream smart_mobile {
server unix:/var/opt/smart-mobile/puma/puma.socket;
}
server {
listen 80;
server_name 10.10.20.108;
access_log /var/log/smart-mobile/nginx/smart-mobile-http.access.log;
error_log /var/log/smart-mobile/nginx/smart-mobile-http.error.log;
root /opt/smart-mobile/embedded/smart-mobile-rails/public;
index index.html;
## Real IP Module Config
## http://nginx.org/en/docs/http/ngx_http_realip_module.html
location / {
if (-f /opt/smart-mobile/embedded/smart-mobile-rails/tmp/maintenance.enable) {
return 503;
}
proxy_http_version 1.1;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_redirect off;
try_files $uri $uri/index.html $uri.html #ruby;
}
location #ruby {
proxy_pass http://smart_mobile;
}
error_page 404 /404.html;
error_page 402 /402.html;
error_page 500 /500.html;
error_page 502 /502.html;
error_page 503 #maintenance;
location #maintenance {
if ($uri !~ ^/icos/) {
rewrite ^(.*)$ /503.html break;
}
}
}
Puma Config:
directory '/opt/smart-mobile/embedded/smart-mobile-rails'
threads 2,4
bind 'unix:///var/opt/smart-mobile/puma/puma.socket'
pidfile '/var/opt/smart-mobile/puma/puma.pid'
preload_app!
on_worker_boot do
ActiveSupport.on_load(:active_record) do
ActiveRecord::Base.establish_connection
end
end
before_fork do
ActiveRecord::Base.connection_pool.disconnect!
end
This worked for me (puma 3.4.0):
# Serve static content if a corresponding file exists.
location / {
try_files $uri #proxy;
# NOTE: Parameters below apply ONLY for static files that match.
expires max;
add_header Cache-Control "public";
add_header By-Nginx "yes"; # DEBUG
}
# Serve dynamic content from the backend.
location #proxy {
proxy_pass http://backend_for_www.site.com;
proxy_pass_request_headers on;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
After some exploration, I've found out that:
Puma is trained to look at HTTP header X-Forwarded-For specifically.
Once it's passed correctly, Puma should hook it up.
No configuration on Puma end is necessary.
request.headers["REMOTE_ADDR"] will stay "127.0.0.1", this doesn't change no matter how hard you try.
Passing header X-Real-IP does not affect the logging issue anyhow.
Basically you can use set_remote_address header: "X-Real-IP" in Puma configuration file to set "remote address of the connection" from this header.
But Puma itself doesn't look in that direction I don't know any other software that does. Documented here: http://www.rubydoc.info/gems/puma/3.2.0/Puma%2FDSL%3Aset_remote_address.
This was my own fault I had all my proxy_set_headers before the try_files. I moved the proxy_set_header directives into the #ruby location block and removed the X-Real-IP header. Everything is working now thank you for all the input.

Ruby process running at max CPU usage and not serving any page - Ubuntu Nginx Unicorn Rails Configuration

I am following this guide to setup Rails service using Nginx and Unicorn http://ariejan.net/2011/09/14/lighting-fast-zero-downtime-deployments-with-git-capistrano-nginx-and-unicorn/
When I started Nginx without Unicorn I get 502 Bad Gateway error
and as soon as I start the Unicorn server using the following command unicorn_rails -c config/unicorn.rb -D the request times out and I get 504 Gateway Time-out error. The CPU usage for ruby process is 100% and seems like something is stuck in a loop but I do not understand what is happening
nginx/1.2.6 (Ubuntu)
This is my /etc/nginx/nginx.conf
user ubuntu staff;
# Change this depending on your hardware
worker_processes 4;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
multi_accept on;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay off;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_disable "msie6";
# gzip_vary on;
gzip_proxied any;
gzip_min_length 500;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml applicat
ion/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
and this is my /etc/nginx/sites-available/default
upstream home {
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response (in case the Unicorn master nukes a
# single worker for timing out).
# for UNIX domain socket setups:
server unix:/tmp/home.socket fail_timeout=0;
}
server {
# if you're running multiple servers, instead of "default" you should
# put your main domain name here
listen 80;
# you could put a list of other domain names this application answers
server_name patellabs.com;
root /home/ubuntu/apps/home/current/public;
access_log /var/log/nginx/home_access.log;
rewrite_log on;
location / {
#all requests are sent to the UNIX socket
proxy_pass http://home;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m;
client_body_buffer_size 128k;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 4k;
proxy_buffers 4 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;
}
# if the request is for a static resource, nginx should serve it directly
# and add a far future expires header to it, making the browser
# cache the resource and navigate faster over the website
# this probably needs some work with Rails 3.1's asset pipe_line
location ~ ^/(images|javascripts|stylesheets|system)/ {
root /home/ubuntu/apps/home/current/public;
expires max;
break;
}
}

Resources