Pushing large docker image to private registry is endless with multiple retryings - docker

Question

Suddenly we got a problem while pushing to our private registry a large docker image.
b09ec9b220b5: Pushing [=======================> ] 1.871GB/3.905GB // always interrupted when reached around 2.3GB/3.905GB
935138875f7e: Layer already exists
844a98711e4c: Layer already exists
f11fbff78593: Layer already exists
cfe955540946: Layer already exists
0e9af88280bb: Layer already exists
08de5656e13a: Layer already exists
5f70bf18a086: Layer already exists
03825ee9b44a: Layer already exists
407fdf993b55: Layer already exists
315572e6ad81: Layer already exists
44dda8402940: Layer already exists
25972ffba58c: Layer already exists
dc6462f7bb8b: Layer already exists
a4db1a405763: Layer already exists
9f4f964da727: Layer already exists
49b333f7bad4: Layer already exists
a463dbda4664: Layer already exists
a9099c3159f5: Layer already exists
Traefik and registry run in docker swarm:
Traefik docker .yml file
version: '3.6'
services:
traefik:
image: traefik:2.9.6
command:
# - '--log.level=DEBUG'
- '--api=true'
- '--api.dashboard=true'
- '--providers.docker.endpoint=unix:///var/run/docker.sock'
- '--providers.docker.swarmMode=true'
- '--providers.docker.exposedbydefault=false'
- '--providers.docker.network=traefik-proxy'
- '--entrypoints.web.address=:80'
- '--entrypoints.websecure.address=:443'
- '--certificatesresolvers.letsencryptresolver.acme.httpchallenge=true'
- '--certificatesresolvers.letsencryptresolver.acme.httpchallenge.entrypoint=web'
- '--certificatesresolvers.letsencryptresolver.acme.email=developer#domain.com'
- '--certificatesresolvers.letsencryptresolver.acme.storage=/letsencrypt/acme.json'
ports:
- "80:80"
- "443:443"
volumes:
- acme-certs:/letsencrypt
- /var/run/docker.sock:/var/run/docker.sock:ro
networks:
- traefik-proxy
deploy:
placement:
constraints:
- node.role == manager
labels:
- 'traefik.enable=true'
# HTTP Catchall for redirecting HTTP -> HTTPS
- 'traefik.http.routers.http-catchall.rule=PathPrefix(`/`)'
- 'traefik.http.routers.http-catchall.entrypoints=web'
- 'traefik.http.routers.http-catchall.middlewares=redirect-to-https'
- 'traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https'
# Traefik dashboard
- 'traefik.http.routers.traefik.rule=Host(`traefik.domain.com`)'
- 'traefik.http.routers.traefik.entrypoints=websecure'
- 'traefik.http.routers.traefik.tls.certresolver=letsencryptresolver'
- 'traefik.http.routers.traefik.service=api#internal'
- 'traefik.http.services.api#internal.loadbalancer.server.port=8080'
- 'traefik.http.routers.traefik.middlewares=traefik-auth'
- 'traefik.http.middlewares.traefik-auth.basicauth.users=admin:$$2y$$10$$OwPg6oWN2iJk5Syde7yIt.tKebbZwtQkaPQ.0AaaaFjNIb1h8bJ6a'
resources:
limits:
memory: "512M"
volumes:
acme-certs:
networks:
traefik-proxy:
external: true
Registry docker .yml file
version: '3.4'
services:
registry:
image: registry:2.8.1
ports:
- "5000:5000"
environment:
- REGISTRY_AUTH=htpasswd
- REGISTRY_STORAGE_DELETE_ENABLED=true
- REGISTRY_AUTH_HTPASSWD_PATH=/httpasswd_storage/htpasswd
- REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm
volumes:
- ./htpasswd_backup:/httpasswd_storage
- registry-data:/var/lib/registry
networks:
- traefik-proxy
- registry-ui-net
deploy:
labels:
- 'traefik.enable=true'
- 'traefik.docker.network=traefik-proxy'
- "traefik.http.routers.registry.rule=Host(`registry1.domain.com`)"
- 'traefik.http.routers.registry.entrypoints=websecure'
- 'traefik.http.routers.registry.tls.certresolver=letsencryptresolver'
- 'traefik.http.services.registry.loadbalancer.server.port=5000'
networks:
traefik-proxy:
external: true
registry-ui-net:
volumes:
registry-data:
There is no any errors in docker registry container logs. When I push this image from my local MacOS - it is endless but if it is pushed from Jenkins (which is on the same server with registry) - there are around 5 retries and then 500 error