I'm trying to set up Hyperledger Sawtooth with 4 nodes in PoET consensus with custom transaction processors connected with each validator with Express App as a proxy to connect with the REST-API engine in docker containers with matrices in Grafana. The validator is not generating the transaction receipt hence, the payload is not processed by my custom Transaction Processor. The batch_status API always gives PENDING status.
Full project link: https://github.com/Shritesh99/Evidence-Management-System
Please Help!!!
Here is my sawtooth-poet.yaml docker-compose file.
version: "3.7"
volumes:
poet-shared:
services:
# -------------=== shell ===-------------
shell:
image: "hyperledger/sawtooth-shell:chime"
container_name: ems-shell
volumes:
- poet-shared:/poet-shared
stop_signal: SIGKILL
entrypoint: |
bash -c "
if [ ! -f /root/.sawtooth/keys/root.priv ]; then
sawtooth keygen
fi;
tail -f /dev/null
"
# -------------=== rest api proxy ===-------------
rest-api-proxy:
build: ./proxy
container_name: ems-rest-api-proxy
volumes:
- ./proxy:/proxy
- /proxy/node_modules
expose:
- 4000
ports:
- "4000:4000"
depends_on:
- rest-api-0
# -------------=== validators ===-------------
validator-0:
image: "hyperledger/sawtooth-validator:chime"
container_name: ems-validator-0
expose:
- 4004
- 5050
- 8800
volumes:
- ./metrices/validator.toml:/etc/sawtooth/validator.toml
- poet-shared:/poet-shared
entrypoint: "bash -c \"\
sawadm keygen --force && \
mkdir -p /poet-shared/validator-0 || true && \
cp -a /etc/sawtooth/keys /poet-shared/validator-0/ && \
while [ ! -f /poet-shared/poet-enclave-measurement ]; do sleep 1; done && \
while [ ! -f /poet-shared/poet-enclave-basename ]; do sleep 1; done && \
while [ ! -f /poet-shared/poet.batch ]; do sleep 1; done && \
cp /poet-shared/poet.batch / && \
sawset genesis \
-k /etc/sawtooth/keys/validator.priv \
-o config-genesis.batch && \
sawset proposal create \
-k /etc/sawtooth/keys/validator.priv \
sawtooth.consensus.algorithm.name=PoET \
sawtooth.consensus.algorithm.version=0.1 \
sawtooth.poet.report_public_key_pem=\
\\\"$$(cat /poet-shared/simulator_rk_pub.pem)\\\" \
sawtooth.poet.valid_enclave_measurements=$$(cat /poet-shared/poet-enclave-measurement) \
sawtooth.poet.valid_enclave_basenames=$$(cat /poet-shared/poet-enclave-basename) \
-o config.batch && \
sawset proposal create \
-k /etc/sawtooth/keys/validator.priv \
sawtooth.poet.target_wait_time=5 \
sawtooth.poet.initial_wait_time=25 \
sawtooth.publisher.max_batches_per_block=100 \
-o poet-settings.batch && \
sawadm genesis \
config-genesis.batch config.batch poet.batch poet-settings.batch && \
sawtooth-validator -v \
--bind network:tcp://eth0:8800 \
--bind component:tcp://eth0:4004 \
--bind consensus:tcp://eth0:5050 \
--peering static \
--endpoint tcp://validator-0:8800 \
--scheduler parallel \
--network-auth trust
\""
environment:
PYTHONPATH: "/project/sawtooth-core/consensus/poet/common:\
/project/sawtooth-core/consensus/poet/simulator:\
/project/sawtooth-core/consensus/poet/core"
stop_signal: SIGKILL
validator-1:
image: "hyperledger/sawtooth-validator:chime"
container_name: ems-validator-1
expose:
- 4004
- 5050
- 8800
volumes:
- ./metrices/validator.toml:/etc/sawtooth/validator.toml
- poet-shared:/poet-shared
entrypoint: |
bash -c "
sawadm keygen --force && \
mkdir -p /poet-shared/validator-1 || true && \
cp -a /etc/sawtooth/keys /poet-shared/validator-1/ && \
sawtooth-validator -v \
--bind network:tcp://eth0:8800 \
--bind component:tcp://eth0:4004 \
--bind consensus:tcp://eth0:5050 \
--peering static \
--endpoint tcp://validator-1:8800 \
--peers tcp://validator-0:8800 \
--scheduler parallel \
--network-auth trust
"
environment:
PYTHONPATH: "/project/sawtooth-core/consensus/poet/common:\
/project/sawtooth-core/consensus/poet/simulator:\
/project/sawtooth-core/consensus/poet/core"
stop_signal: SIGKILL
validator-2:
image: "hyperledger/sawtooth-validator:chime"
container_name: ems-validator-2
expose:
- 4004
- 5050
- 8800
volumes:
- ./metrices/validator.toml:/etc/sawtooth/validator.toml
- poet-shared:/poet-shared
entrypoint: |
bash -c "
sawadm keygen --force && \
mkdir -p /poet-shared/validator-2 && \
cp -a /etc/sawtooth/keys /poet-shared/validator-2/ && \
sawtooth-validator -v \
--bind network:tcp://eth0:8800 \
--bind component:tcp://eth0:4004 \
--bind consensus:tcp://eth0:5050 \
--peering static \
--endpoint tcp://validator-2:8800 \
--peers tcp://validator-0:8800,tcp://validator-1:8800 \
--scheduler parallel \
--network-auth trust
"
environment:
PYTHONPATH: "/project/sawtooth-core/consensus/poet/common:\
/project/sawtooth-core/consensus/poet/simulator:\
/project/sawtooth-core/consensus/poet/core"
stop_signal: SIGKILL
validator-3:
image: "hyperledger/sawtooth-validator:chime"
container_name: ems-validator-3
expose:
- 4004
- 5050
- 8800
volumes:
- ./metrices/validator.toml:/etc/sawtooth/validator.toml
- poet-shared:/poet-shared
entrypoint: |
bash -c "
sawadm keygen --force && \
mkdir -p /poet-shared/validator-3 && \
cp -a /etc/sawtooth/keys /poet-shared/validator-3/ && \
sawtooth-validator -v \
--bind network:tcp://eth0:8800 \
--bind component:tcp://eth0:4004 \
--bind consensus:tcp://eth0:5050 \
--peering static \
--endpoint tcp://validator-3:8800 \
--peers tcp://validator-0:8800,tcp://validator-1:8800,tcp://validator-2:8800 \
--scheduler parallel \
--network-auth trust
"
environment:
PYTHONPATH: "/project/sawtooth-core/consensus/poet/common:\
/project/sawtooth-core/consensus/poet/simulator:\
/project/sawtooth-core/consensus/poet/core"
stop_signal: SIGKILL
validator-4:
image: "hyperledger/sawtooth-validator:chime"
container_name: ems-validator-4
expose:
- 4004
- 5050
- 8800
volumes:
- ./metrices/validator.toml:/etc/sawtooth/validator.toml
- poet-shared:/poet-shared
entrypoint: |
bash -c "
sawadm keygen --force && \
mkdir -p /poet-shared/validator-4 && \
cp -a /etc/sawtooth/keys /poet-shared/validator-4/ && \
sawtooth-validator -v \
--bind network:tcp://eth0:8800 \
--bind component:tcp://eth0:4004 \
--bind consensus:tcp://eth0:5050 \
--peering static \
--endpoint tcp://validator-4:8800 \
--peers tcp://validator-0:8800,tcp://validator-1:8800,tcp://validator-2:8800,tcp://validator-3:8800 \
--scheduler parallel \
--network-auth trust
"
environment:
PYTHONPATH: "/project/sawtooth-core/consensus/poet/common:\
/project/sawtooth-core/consensus/poet/simulator:\
/project/sawtooth-core/consensus/poet/core"
stop_signal: SIGKILL
# -------------=== rest api ===-------------
rest-api-0:
image: "hyperledger/sawtooth-rest-api:chime"
container_name: ems-rest-api-0
volumes:
- ./metrices/rest_api.toml:/etc/sawtooth/rest_api.toml
depends_on:
- validator-0
entrypoint: "sawtooth-rest-api -C tcp://validator-0:4004 --bind rest-api-0:8008 -v"
stop_signal: SIGKILL
rest-api-1:
image: "hyperledger/sawtooth-rest-api:chime"
container_name: ems-rest-api-1
volumes:
- ./metrices/rest_api.toml:/etc/sawtooth/rest_api.toml
depends_on:
- validator-1
entrypoint: "sawtooth-rest-api -C tcp://validator-1:4004 --bind rest-api-1:8008 -v"
stop_signal: SIGKILL
rest-api-2:
image: "hyperledger/sawtooth-rest-api:chime"
container_name: ems-rest-api-2
volumes:
- ./metrices/rest_api.toml:/etc/sawtooth/rest_api.toml
depends_on:
- validator-2
entrypoint: "sawtooth-rest-api -C tcp://validator-2:4004 --bind rest-api-2:8008 -v"
stop_signal: SIGKILL
rest-api-3:
image: "hyperledger/sawtooth-rest-api:chime"
container_name: ems-rest-api-3
volumes:
- ./metrices/rest_api.toml:/etc/sawtooth/rest_api.toml
depends_on:
- validator-3
entrypoint: "sawtooth-rest-api -C tcp://validator-3:4004 --bind rest-api-3:8008 -v"
stop_signal: SIGKILL
rest-api-4:
image: "hyperledger/sawtooth-rest-api:chime"
container_name: ems-rest-api-4
volumes:
- ./metrices/rest_api.toml:/etc/sawtooth/rest_api.toml
depends_on:
- validator-4
entrypoint: "sawtooth-rest-api -C tcp://validator-4:4004 --bind rest-api-4:8008 -v"
stop_signal: SIGKILL
# -------------=== settings tp ===-------------
settings-tp-0:
image: "hyperledger/sawtooth-settings-tp:chime"
container_name: ems-settings-tp-0
depends_on:
- validator-0
entrypoint: "settings-tp -v -C tcp://validator-0:4004"
stop_signal: SIGKILL
settings-tp-1:
image: "hyperledger/sawtooth-settings-tp:chime"
container_name: ems-settings-tp-1
depends_on:
- validator-1
entrypoint: "settings-tp -v -C tcp://validator-1:4004"
stop_signal: SIGKILL
settings-tp-2:
image: "hyperledger/sawtooth-settings-tp:chime"
container_name: ems-settings-tp-2
depends_on:
- validator-2
entrypoint: "settings-tp -v -C tcp://validator-2:4004"
stop_signal: SIGKILL
settings-tp-3:
image: "hyperledger/sawtooth-settings-tp:chime"
container_name: ems-settings-tp-3
depends_on:
- validator-3
entrypoint: "settings-tp -v -C tcp://validator-3:4004"
stop_signal: SIGKILL
settings-tp-4:
image: "hyperledger/sawtooth-settings-tp:chime"
container_name: ems-settings-tp-4
depends_on:
- validator-4
entrypoint: "settings-tp -v -C tcp://validator-4:4004"
stop_signal: SIGKILL
# -------------=== poet engines ===-------------
poet-engine-0:
image: "hyperledger/sawtooth-poet-engine:chime"
container_name: ems-poet-engine-0
volumes:
- poet-shared:/poet-shared
depends_on:
- validator-0
entrypoint: |
bash -c "
if [ ! -f /poet-shared/poet-enclave-measurement ]; then
poet enclave measurement >> /poet-shared/poet-enclave-measurement; \
fi &&
if [ ! -f /poet-shared/poet-enclave-basename ]; then
poet enclave basename >> /poet-shared/poet-enclave-basename;
fi &&
if [ ! -f /poet-shared/simulator_rk_pub.pem ]; then
cp /etc/sawtooth/simulator_rk_pub.pem /poet-shared;
fi &&
while [ ! -f /poet-shared/validator-0/keys/validator.priv ];
do sleep 1; done
cp -a /poet-shared/validator-0/keys /etc/sawtooth &&
poet registration create -k /etc/sawtooth/keys/validator.priv -o /poet-shared/poet.batch &&
poet-engine -C tcp://validator-0:5050 --component tcp://validator-0:4004
"
poet-engine-1:
image: "hyperledger/sawtooth-poet-engine:chime"
container_name: ems-poet-engine-1
volumes:
- poet-shared:/poet-shared
depends_on:
- validator-1
entrypoint: |
bash -c "
while [ ! -f /poet-shared/validator-1/keys/validator.priv ];
do sleep 1; done
cp -a /poet-shared/validator-1/keys /etc/sawtooth &&
poet-engine -C tcp://validator-1:5050 --component tcp://validator-1:4004
"
poet-engine-2:
image: "hyperledger/sawtooth-poet-engine:chime"
container_name: ems-poet-engine-2
volumes:
- poet-shared:/poet-shared
depends_on:
- validator-2
entrypoint: |
bash -c "
while [ ! -f /poet-shared/validator-2/keys/validator.priv ];
do sleep 1; done
cp -a /poet-shared/validator-2/keys /etc/sawtooth &&
poet-engine -C tcp://validator-2:5050 --component tcp://validator-2:4004
"
poet-engine-3:
image: "hyperledger/sawtooth-poet-engine:chime"
container_name: ems-poet-engine-3
volumes:
- poet-shared:/poet-shared
depends_on:
- validator-3
entrypoint: |
bash -c "
while [ ! -f /poet-shared/validator-3/keys/validator.priv ];
do sleep 1; done
cp -a /poet-shared/validator-3/keys /etc/sawtooth &&
poet-engine -C tcp://validator-3:5050 --component tcp://validator-3:4004
"
poet-engine-4:
image: "hyperledger/sawtooth-poet-engine:chime"
container_name: ems-poet-engine-4
volumes:
- poet-shared:/poet-shared
depends_on:
- validator-4
entrypoint: |
bash -c "
while [ ! -f /poet-shared/validator-4/keys/validator.priv ];
do sleep 1; done
cp -a /poet-shared/validator-4/keys /etc/sawtooth &&
poet-engine -C tcp://validator-4:5050 --component tcp://validator-4:4004
"
# -------------=== poet validator registry ===-------------
poet-validator-registry-tp-0:
image: "hyperledger/sawtooth-poet-validator-registry-tp:chime"
container_name: ems-poet-validator-registry-tp-0
depends_on:
- validator-0
entrypoint: "poet-validator-registry-tp -C tcp://validator-0:4004 -v"
environment:
PYTHONPATH: /project/sawtooth-core/consensus/poet/common
stop_signal: SIGKILL
poet-validator-registry-tp-1:
image: "hyperledger/sawtooth-poet-validator-registry-tp:chime"
container_name: ems-poet-validator-registry-tp-1
depends_on:
- validator-1
entrypoint: "poet-validator-registry-tp -C tcp://validator-1:4004 -v"
environment:
PYTHONPATH: /project/sawtooth-core/consensus/poet/common
stop_signal: SIGKILL
poet-validator-registry-tp-2:
image: "hyperledger/sawtooth-poet-validator-registry-tp:chime"
container_name: ems-poet-validator-registry-tp-2
depends_on:
- validator-2
entrypoint: "poet-validator-registry-tp -C tcp://validator-2:4004 -v"
environment:
PYTHONPATH: /project/sawtooth-core/consensus/poet/common
stop_signal: SIGKILL
poet-validator-registry-tp-3:
image: "hyperledger/sawtooth-poet-validator-registry-tp:chime"
container_name: ems-poet-validator-registry-tp-3
depends_on:
- validator-3
entrypoint: "poet-validator-registry-tp -C tcp://validator-3:4004 -v"
environment:
PYTHONPATH: /project/sawtooth-core/consensus/poet/common
stop_signal: SIGKILL
poet-validator-registry-tp-4:
image: "hyperledger/sawtooth-poet-validator-registry-tp:chime"
container_name: ems-poet-validator-registry-tp-4
depends_on:
- validator-4
entrypoint: "poet-validator-registry-tp -C tcp://validator-4:4004 -v"
environment:
PYTHONPATH: /project/sawtooth-core/consensus/poet/common
stop_signal: SIGKILL
# -------------=== EMS tps ===-------------
processor-0:
image: processor
container_name: ems-processor-0
build: ./processor
volumes:
- ./processor:/processor
- /processor/node_modules
depends_on:
- validator-0
environment:
- VALIDATOR_URL=tcp://validator-0:4004
processor-1:
image: processor
container_name: ems-processor-1
build: ./processor
volumes:
- ./processor:/processor
- /processor/node_modules
depends_on:
- validator-1
environment:
- VALIDATOR_URL=tcp://validator-1:4004
processor-2:
image: processor
container_name: ems-processor-2
build: ./processor
volumes:
- ./processor:/processor
- /processor/node_modules
depends_on:
- validator-2
environment:
- VALIDATOR_URL=tcp://validator-2:4004
processor-3:
image: processor
container_name: ems-processor-3
build: ./processor
volumes:
- ./processor:/processor
- /processor/node_modules
depends_on:
- validator-3
environment:
- VALIDATOR_URL=tcp://validator-3:4004
processor-4:
image: processor
container_name: ems-processor-4
build: ./processor
volumes:
- ./processor:/processor
- /processor/node_modules
depends_on:
- validator-4
environment:
- VALIDATOR_URL=tcp://validator-4:4004
Here are my validator-0 logs.
writing file: /etc/sawtooth/keys/validator.priv
writing file: /etc/sawtooth/keys/validator.pub
Generated config-genesis.batch
Processing config-genesis.batch...
Processing config.batch...
Processing poet.batch...
Processing poet-settings.batch...
Generating /var/lib/sawtooth/genesis.batch
[2020-12-02 14:04:18.313 WARNING (unknown file)] [src/pylogger.rs: 40] Started logger at level INFO
[2020-12-02 14:04:19.347 INFO path] Skipping path loading from non-existent config file: /etc/sawtooth/path.toml
[2020-12-02 14:04:19.347 INFO validator] Loading validator information from config: /etc/sawtooth/validator.toml
[2020-12-02 14:04:19.354 INFO path] Skipping path loading from non-existent config file: /etc/sawtooth/path.toml
[2020-12-02 14:04:19.356 INFO keys] Loading signing key: /etc/sawtooth/keys/validator.priv
[2020-12-02 14:04:19.360 INFO cli] sawtooth-validator (Hyperledger Sawtooth) version 1.2.5
[2020-12-02 14:04:19.360 INFO cli] config [path]: config_dir = "/etc/sawtooth"; config [path]: key_dir = "/etc/sawtooth/keys"; config [path]: data_dir = "/var/lib/sawtooth"; config [path]: log_dir = "/var/log/sawtooth"; config [path]: policy_dir = "/etc/sawtooth/policy"
[2020-12-02 14:04:19.361 INFO cli] Adding metrics reporter: url=http://192.168.29.153:8086, db=metrics
[2020-12-02 14:04:19.407 INFO state_verifier] Skipping state verification: chain head's state root is present
[2020-12-02 14:04:19.410 INFO cli] Starting validator with parallel scheduler
[2020-12-02 14:04:19.491 INFO interconnect] Listening on tcp://eth0:4004
[2020-12-02 14:04:19.496 INFO genesis] Producing genesis block from /var/lib/sawtooth/genesis.batch
[2020-12-02 14:04:19.502 INFO processor_manager] waiting for processor type sawtooth_settings: 1.0 to register
[2020-12-02 14:04:29.473 WARNING influx] Cannot write to 192.168.29.153: Bad Request
[2020-12-02 14:04:30.070 WARNING processor_handlers] Max occupancy was not provided by transaction processor: 5ff1b5d1495bc765a3b0436db794658cfcf0ce6af8875ed448a2db8a37d8e8103aab52e10cc7ffb17b35e27c45dfa2df38548f5393aefce91b369ba4ef622bad. Using default max occupancy: 10
[2020-12-02 14:04:30.070 INFO processor_handlers] registered transaction processor: connection_id=5ff1b5d1495bc765a3b0436db794658cfcf0ce6af8875ed448a2db8a37d8e8103aab52e10cc7ffb17b35e27c45dfa2df38548f5393aefce91b369ba4ef622bad, family=sawtooth_settings, version=1.0, namespaces=['000000'], max_occupancy=10
[2020-12-02 14:04:30.591 INFO processor_manager] waiting for processor type sawtooth_validator_registry: 1.0 to register
[2020-12-02 14:04:30.997 WARNING processor_handlers] Max occupancy was not provided by transaction processor: aaf4f8f70f7f076a594e63a7c3b81ca26f8658ceb76e763462a7edb8bcfdeb3eec095444c732628130ba13d2d0e73f4633c6e58be636c11c0fef25c016df27c3. Using default max occupancy: 10
[2020-12-02 14:04:31.001 INFO processor_handlers] registered transaction processor: connection_id=aaf4f8f70f7f076a594e63a7c3b81ca26f8658ceb76e763462a7edb8bcfdeb3eec095444c732628130ba13d2d0e73f4633c6e58be636c11c0fef25c016df27c3, family=sawtooth_validator_registry, version=1.0, namespaces=['6a4372'], max_occupancy=10
[2020-12-02 14:04:31.162 WARNING processor_handlers] Max occupancy was not provided by transaction processor: b0234249154584f591b355b68a48bc12ad73dfd0f08bd751bdcc640ed8fd0be3ccfbda04e723b3bb1db017533df7c49bbf64b9bfe3864da3c6e2f19d4adda2c9. Using default max occupancy: 10
[2020-12-02 14:04:31.163 INFO processor_handlers] registered transaction processor: connection_id=b0234249154584f591b355b68a48bc12ad73dfd0f08bd751bdcc640ed8fd0be3ccfbda04e723b3bb1db017533df7c49bbf64b9bfe3864da3c6e2f19d4adda2c9, family=evidence_management_system, version=0.0, namespaces=['d23299'], max_occupancy=10
[2020-12-02 14:04:32.595 INFO genesis] Genesis block created: 133d0d3c7d345c1f5e62de798fd698869a050b895f4df0531a69ddcbf3242eb820a08e4dd4329927c36ba0bf9aac605d511395e5ce51196e2737b26d0d7e061e (block_num:0, state:c8e91ecf6cebdc28b648429b585fba559368c7c1616e4e50ecbd35169ab94113, previous_block_id:0000000000000000)
[2020-12-02 14:04:32.679 INFO interconnect] Listening on tcp://eth0:5050
[2020-12-02 14:04:32.809 INFO interconnect] Listening on tcp://eth0:8800
[2020-12-02 14:04:32.872 INFO ffi] [src/journal/chain.rs: 946] Chain controller initialized with chain head: Block(id: 133d0d3c7d345c1f5e62de798fd698869a050b895f4df0531a69ddcbf3242eb820a08e4dd4329927c36ba0bf9aac605d511395e5ce51196e2737b26d0d7e061e, block_num: 0, state_root_hash: c8e91ecf6cebdc28b648429b585fba559368c7c1616e4e50ecbd35169ab94113, previous_block_id: 0000000000000000)
[2020-12-02 14:04:32.877 INFO ffi] [src/journal/publisher.rs: 172] Now building on top of block, Block(id: 133d0d3c7d345c1f5e62de798fd698869a050b895f4df0531a69ddcbf3242eb820a08e4dd4329927c36ba0bf9aac605d511395e5ce51196e2737b26d0d7e061e, block_num: 0, state_root_hash: c8e91ecf6cebdc28b648429b585fba559368c7c1616e4e50ecbd35169ab94113, previous_block_id: 0000000000000000)
[2020-12-02 14:04:39.123 INFO handlers] Consensus engine registered: PoET 0.1 (additional protocols: [name: "poet"
version: "0.1"
])
[2020-12-02 14:04:39.137 INFO proxy] Consensus engine activated: PoET 0.1
Here is my REST-API-0 logs.
[2020-12-02 14:04:36.477 INFO rest_api] Adding metrics reporter: url=http://192.168.29.153:8086, db=metrics
[2020-12-02 14:04:36.600 INFO messaging] Connecting to tcp://validator-0:4004
[2020-12-02 14:04:36.613 INFO rest_api] Creating handlers for validator at tcp://validator-0:4004
[2020-12-02 14:04:36.823 INFO rest_api] Starting REST API on rest-api-0:8008
[2020-12-02 14:04:40.635 INFO helpers] GET /batch_statuses?id=c965b70251bdb8005547812ae8655ebd43421e30b57602eb550f08cf119ba8c002084e89449c8130c94de8865a58eec2e7741e839168c570f9fb4a55659ed8be HTTP/1.1: 200 status, 594 size, in 0.018432 s
[2020-12-02 14:05:55.055 INFO helpers] OPTIONS /batches HTTP/1.1: 405 status, 231 size, in 0.000543 s
[2020-12-02 14:05:55.107 INFO helpers] POST /batches HTTP/1.1: 202 status, 367 size, in 0.020589 s
[2020-12-02 14:06:00.151 INFO helpers] GET /batch_statuses?id=05a44cae97e4356a104133391d1cb09c92c5c870dc50712738430107b505beec6baee39a0e2efabbc3d6dac2fea2b83c3edbcfce58ee2467e60555d81f0e8a09 HTTP/1.1: 200 status, 594 size, in 0.003084 s
[2020-12-02 14:06:05.166 INFO helpers] GET /batch_statuses?id=05a44cae97e4356a104133391d1cb09c92c5c870dc50712738430107b505beec6baee39a0e2efabbc3d6dac2fea2b83c3edbcfce58ee2467e60555d81f0e8a09 HTTP/1.1: 200 status, 594 size, in 0.003312 s
Here are Custom transaction processor's logs.
Initializing EMS handler for Evidence Management System
Connected to tcp://validator-0:4004
Registration of [evidence_management_system 0.0] succeeded
Received Ping
Received Ping
Edit - 1
Here is PoET-Engine-0 logs:
Writing key state for PoET public key: 027cb2ab...e715c7dc
Generating /poet-shared/poet.batch
Related
I am attempting to add a network drive for reading and writing in Airflow. I am able to read from the directory, however when I attempt to write to the directory with a DAG I receive a "Permission Denied" error. Is there a permissions issue I need to fix in order to write to this directory?
---
version: '3'
x-airflow-common:
&airflow-common
# In order to add custom dependencies or upgrade provider packages you can use your extended image.
# Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
# and uncomment the "build" line below, Then run `docker-compose build` to build the images.
#image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.3.3}
build: .
env_file:
- .env
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow#postgres/airflow
# For backward compatibility, with Airflow <2.3
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow#postgres/airflow
AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow#postgres/airflow
AIRFLOW__CELERY__BROKER_URL: redis://:#redis:6379/0
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth'
AIRFLOW__EMAIL__EMAIL_BACKEND: 'airflow.utils.email.send_email_smtp'
AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_RETRY: True
AIRFLOW__EMAIL__DEFAULT_EMAIL_ON_FAILURE: True
AIRFLOW__SMTP__SMTP_HOST: 'mail.precorp.org'
AIRFLOW__SMTP__SMTP_STARTTLS: True
AIRFLOW__SMTP__SMTP_SSL: False
AIRFLOW__SMTP__SMTP_PORT: 25
AIRFLOW__SMTP__SMTP_MAIL_FROM: 'gis#precorp.coop'
AIRFLOW__WEBSERVER__BASE_URL: 'http://10.54.0.25:8080'
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
volumes:
- ./dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- gissa:/gissa
user: "${AIRFLOW_UID:-50000}:0"
depends_on:
&airflow-common-depends-on
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:13
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 5s
retries: 5
restart: always
redis:
image: redis:latest
expose:
- 6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<<: *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-scheduler:
<<: *airflow-common
command: scheduler
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-worker:
<<: *airflow-common
command: celery worker
healthcheck:
test:
- "CMD-SHELL"
- 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery#$${HOSTNAME}"'
interval: 10s
timeout: 10s
retries: 5
env_file:
- .env
environment:
<<: *airflow-common-env
# Required to handle warm shutdown of the celery workers properly
# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
DUMB_INIT_SETSID: "0"
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-triggerer:
<<: *airflow-common
command: triggerer
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-init:
<<: *airflow-common
entrypoint: /bin/bash
# yamllint disable rule:line-length
command:
- -c
- |
function ver() {
printf "%04d%04d%04d%04d" $${1//./ }
}
airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version)
airflow_version_comparable=$$(ver $${airflow_version})
min_airflow_version=2.2.0
min_airflow_version_comparable=$$(ver $${min_airflow_version})
if (( airflow_version_comparable < min_airflow_version_comparable )); then
echo
echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
echo
exit 1
fi
if [[ -z "${AIRFLOW_UID}" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
echo "If you are on Linux, you SHOULD follow the instructions below to set "
echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
echo "For other operating systems you can get rid of the warning with manually created .env file:"
echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user"
echo
fi
one_meg=1048576
mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
disk_available=$$(df / | tail -1 | awk '{print $$4}')
warning_resources="false"
if (( mem_available < 4000 )) ; then
echo
echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
echo
warning_resources="true"
fi
if (( cpus_available < 2 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
echo "At least 2 CPUs recommended. You have $${cpus_available}"
echo
warning_resources="true"
fi
if (( disk_available < one_meg * 10 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
echo
warning_resources="true"
fi
if [[ $${warning_resources} == "true" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
echo "Please follow the instructions to increase amount of resources available:"
echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin"
echo
fi
mkdir -p /sources/logs /sources/dags /sources/plugins
chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
chown -R "${AIRFLOW_UID}:0" /gissa
exec /entrypoint airflow version
# yamllint enable rule:line-length
env_file:
- .env
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
_PIP_ADDITIONAL_REQUIREMENTS: ''
user: "0:0"
volumes:
- .:/sources
airflow-cli:
<<: *airflow-common
profiles:
- debug
env_file:
- .env
environment:
<<: *airflow-common-env
CONNECTION_CHECK_MAX_COUNT: "0"
# Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
command:
- bash
- -c
- airflow
# You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up
# or by explicitly targeted on the command line e.g. docker-compose up flower.
# See: https://docs.docker.com/compose/profiles/
flower:
<<: *airflow-common
command: celery flower
profiles:
- flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
volumes:
gissa:
driver: local
driver_opts:
type: cifs
o: "username=${NETWORKDRIVE_USER},password=${NETWORKDRIVE_PWD},uid=${AIRFLOW_UID},gid=${AIRFLOW_GID},forceuid,forcegid"
device: "//NETWORKDRIVE/FOLDER_IN_NETWORK_DRIV
postgres-db-volume:
I also tried creating /gissa in my dockerfile with open permissions:
FROM apache/airflow:2.3.3
COPY requirements.txt .
USER root
# Install ODBC-driver 17
RUN apt-get update \
&& apt-get install -y curl apt-transport-https gnupg2 \
&& curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
&& curl https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql-release.list \
&& apt-get update \
&& ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools \
&& apt-get install unixodbc-dev -y
RUN apt-get install -y libgdal-dev g++ --no-install-recommends && \
apt-get clean -y
ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
ENV C_INCLUDE_PATH=/usr/include/gdal
RUN mkdir /gissa
RUN chmod -R 777 /gissa
USER airflow
RUN pip install GDAL==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}')
RUN pip install -r requirements.txt
RUN pip install apache-airflow-providers-microsoft-mssql
I am having trouble copying a file to a container via its dockerfile.
the dockerfile:
FROM tiangolo/uwsgi-nginx-flask:python3.6
LABEL maintainer="Simranjeet Singh <simranjeetsingh1497#gmail.com>"
COPY ./app_dash/requirements.txt /tmp/
COPY ./app_dash /app
COPY ./dags/helpfiles/ml_pipeline_config.py /app
RUN pip install -U pip && pip install -r /tmp/requirements.txt
ENV NGINX_WORKER_PROCESSES auto
Both folders app_dash and dags are located in the folder of my dockerfile. I also get no error message during the build, all of this works. However, if I check the container content itself via cmd, ml_pipeline_config.py is not there. ./app_dash is copied correctly.
Edit: the container is run with the following .yaml (see dash all the way down) :
.yaml and .dockerfile are in the same folder
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Original version: https://airflow.apache.org/docs/apache-airflow/stable/docker-compose.yaml
---
version: '3'
x-airflow-common:
&airflow-common
# In order to add custom dependencies or upgrade provider packages you can use your extended image.
# Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
# and uncomment the "build" line below, Then run `docker-compose build` to build the images.
image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.2}
# build: .
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow#postgres/airflow
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-scikit-learn}
volumes:
- ./dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- ./data:/opt/airflow/data
- ./models:/opt/airflow/models
user: "${AIRFLOW_UID:-50000}:0"
depends_on:
&airflow-common-depends-on
postgres:
condition: service_healthy
services:
postgres:
image: postgres:13
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- postgres-db-volume:/var/lib/postgresql/data
- './data:/maindata' #this was added to mirror the data files into the postgres container. Only then, the much more efficient copy sql command can be used.
ports:
- 5432:5432
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 5s
retries: 5
restart: always
airflow-webserver:
<<: *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-scheduler:
<<: *airflow-common
command: scheduler
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-triggerer:
<<: *airflow-common
command: triggerer
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
interval: 10s
timeout: 10s
retries: 5
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
airflow-init:
<<: *airflow-common
entrypoint: /bin/bash
# yamllint disable rule:line-length
command:
- -c
- |
function ver() {
printf "%04d%04d%04d%04d" $${1//./ }
}
airflow_version=$$(gosu airflow airflow version)
airflow_version_comparable=$$(ver $${airflow_version})
min_airflow_version=2.2.0
min_airflow_version_comparable=$$(ver $${min_airflow_version})
if (( airflow_version_comparable < min_airflow_version_comparable )); then
echo
echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
echo
exit 1
fi
if [[ -z "${AIRFLOW_UID}" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
echo "If you are on Linux, you SHOULD follow the instructions below to set "
echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
echo "For other operating systems you can get rid of the warning with manually created .env file:"
echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user"
echo
fi
one_meg=1048576
mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
disk_available=$$(df / | tail -1 | awk '{print $$4}')
warning_resources="false"
if (( mem_available < 4000 )) ; then
echo
echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
echo
warning_resources="true"
fi
if (( cpus_available < 2 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
echo "At least 2 CPUs recommended. You have $${cpus_available}"
echo
warning_resources="true"
fi
if (( disk_available < one_meg * 10 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
echo
warning_resources="true"
fi
if [[ $${warning_resources} == "true" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
echo "Please follow the instructions to increase amount of resources available:"
echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin"
echo
fi
mkdir -p /sources/logs /sources/dags /sources/plugins
chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
exec /entrypoint airflow version
# yamllint enable rule:line-length
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
user: "0:0"
volumes:
- .:/sources
airflow-cli:
<<: *airflow-common
profiles:
- debug
environment:
<<: *airflow-common-env
CONNECTION_CHECK_MAX_COUNT: "0"
# Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
command:
- bash
- -c
- airflow
dash:
build: .
ports:
- 80:80
volumes:
- ./data:/app #todo: check whether this is needed
- ./app_dash:/app
- /dags/helpfiles/ml_pipeline_config.py /app
depends_on:
postgres:
condition: service_healthy
volumes:
postgres-db-volume: #todo: Check why this is standing here
Where is my mistake?
Thanks already!
The COPY command accept only relative paths, I suggest you to rewrite the docker file as:
FROM tiangolo/uwsgi-nginx-flask:python3.6
LABEL maintainer="Simranjeet Singh <simranjeetsingh1497#gmail.com>"
COPY app_dash/requirements.txt /tmp/
COPY app_dash /app
COPY dags/helpfiles/ml_pipeline_config.py /app
RUN pip install -U pip && pip install -r /tmp/requirements.txt
ENV NGINX_WORKER_PROCESSES auto
The relative path starts from the context path you provide to the docker build command, so you should do:
cd <docker-file-folder>
docker build .
So all the file are picked from the . (current) context folder.
Let me know if it worked.
Regards.
I used the following procedure to install Elastic Search and Kibana 8 RC2 :
# create the data directory on host for persistence
mkdir -p /data/elasticsearch/data
chmod 777 -R /data/elasticsearch/data
# create the elastic network
docker network create elastic
# run elastic search in background with data persistence in a single node configuration and test password
docker run -d -it --name els01 --net elastic -p 0.0.0.0:9200:9200 -p 0.0.0.0:9300:9300 -v /data/elasticsearch/data:/data/elasticsearch/data -e "discovery.type=single-node" -e ELASTIC_PASSWORD="test" -e KIBANA_PASSWORD="test" docker.elastic.co/elasticsearch/elasticsearch:8.0.0-rc2
# run kibana in background
docker run -d -it --name kib01 --net elastic -p 0.0.0.0:5601:5601 -e KIBANA_PASSWORD="test" -e "ELASTICSEARCH_HOSTS=https://els01:9200" docker.elastic.co/kibana/kibana:8.0.0-rc2
On the webpage http://10.0.2.1:5601/, I get this error :
Kibana server is not ready yet.
docker logs --tail 50 --follow --timestamps f82efc804e9c returns this error :
Unable to retrieve version information from Elasticsearch nodes. self signed certificate in certificate chain
What do I need to change in order to have a functionnal Kibana on Elastic Search ?
Thanks
TLDR;
Elasticsearch 8 comes with SSL/TLS enabled by default
Kibana has to have the CA certificate to verify and connect to elasticsearch.
To solve
How about you use the docker-compose file provided to set up a multi-cluster node. It will take care of the SSL/TLS certificate.
You will just need to
delete services es02 and es03
update volumes path
to be back on your actual set up.
This could look like
version: "2.2"
services:
setup:
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
user: "0"
command: >
bash -c '
if [ x${ELASTIC_PASSWORD} == x ]; then
echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
exit 1;
elif [ x${KIBANA_PASSWORD} == x ]; then
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
exit 1;
fi;
if [ ! -f certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f certs/certs.zip ]; then
echo "Creating certs";
echo -ne \
"instances:\n"\
" - name: es01\n"\
" dns:\n"\
" - es01\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 \{\} \;;
find . -type f -exec chmod 640 \{\} \;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "Setting kibana_system password";
until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
echo "All done!";
'
healthcheck:
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
interval: 1s
timeout: 5s
retries: 120
es01:
depends_on:
setup:
condition: service_healthy
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
volumes:
- certs:/usr/share/elasticsearch/config/certs
- type: bind
source: /data/elasticsearch/data
target: /usr/share/elasticsearch/data
ports:
- ${ES_PORT}:9200
environment:
- node.name=es01
- cluster.name=${CLUSTER_NAME}
- cluster.initial_master_nodes=es01,es02,es03
- discovery.seed_hosts=es02,es03
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es01/es01.key
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.http.ssl.verification_mode=certificate
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es01/es01.key
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=${LICENSE}
mem_limit: ${MEM_LIMIT}
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
kibana:
depends_on:
es01:
condition: service_healthy
es02:
condition: service_healthy
es03:
condition: service_healthy
image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
volumes:
- certs:/usr/share/kibana/config/certs
- kibanadata:/usr/share/kibana/data
ports:
- ${KIBANA_PORT}:5601
environment:
- SERVERNAME=kibana
- ELASTICSEARCH_HOSTS=https://es01:9200
- ELASTICSEARCH_USERNAME=kibana_system
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
mem_limit: ${MEM_LIMIT}
healthcheck:
test:
[
"CMD-SHELL",
"curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
]
interval: 10s
timeout: 10s
retries: 120
volumes:
certs:
driver: local
esdata01:
driver: local
kibanadata:
driver: local
My version for single node configuration and without env variables:
version: "2.2"
services:
setup:
image: docker.elastic.co/elasticsearch/elasticsearch:8.3.3
volumes:
- ./volumes/data/es/certs:/usr/share/elasticsearch/config/certs
user: "0"
command: >
bash -c '
if [ ! -f certs/ca.zip ]; then
echo "Creating CA";
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
unzip config/certs/ca.zip -d config/certs;
fi;
if [ ! -f certs/certs.zip ]; then
echo "Creating certs";
echo -ne \
"instances:\n"\
" - name: es01\n"\
" dns:\n"\
" - es01\n"\
" - localhost\n"\
" ip:\n"\
" - 127.0.0.1\n"\
> config/certs/instances.yml;
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
unzip config/certs/certs.zip -d config/certs;
fi;
echo "Setting file permissions"
chown -R root:root config/certs;
find . -type d -exec chmod 750 \{\} \;;
find . -type f -exec chmod 640 \{\} \;;
echo "Waiting for Elasticsearch availability";
until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
echo "Setting kibana_system password";
until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:E0glIRkUbVak8f4aHZe -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"E0glIRkUbVak8f4aHZk\"}" | grep -q "^{}"; do sleep 10; done;
echo "All done!";
'
healthcheck:
test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
interval: 1s
timeout: 5s
retries: 120
es01:
depends_on:
setup:
condition: service_healthy
image: docker.elastic.co/elasticsearch/elasticsearch:8.3.3
volumes:
- ./volumes/data/es/certs:/usr/share/elasticsearch/config/certs
- ./volumes/data/es/es01:/usr/share/elasticsearch/data
ports:
- 9200:9200
environment:
- node.name=es01
- discovery.type=single-node
- ELASTIC_PASSWORD=<ELASTIC_PASSWORD>
- bootstrap.memory_lock=true
- xpack.security.enabled=true
- xpack.security.http.ssl.enabled=true
- xpack.security.http.ssl.key=certs/es01/es01.key
- xpack.security.http.ssl.certificate=certs/es01/es01.crt
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.http.ssl.verification_mode=certificate
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.key=certs/es01/es01.key
- xpack.security.transport.ssl.certificate=certs/es01/es01.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.license.self_generated.type=basic
mem_limit: 1073741824
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
test:
[
"CMD-SHELL",
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
]
interval: 10s
timeout: 10s
retries: 120
kibana:
depends_on:
es01:
condition: service_healthy
image: docker.elastic.co/kibana/kibana:8.3.3
volumes:
- ./volumes/data/es/certs:/usr/share/kibana/config/certs
- ./volumes/data/es/kibanadata:/usr/share/kibana/data
ports:
- 5601:5601
environment:
- SERVERNAME=kibana
- ELASTICSEARCH_HOSTS=https://es01:9200
- ELASTICSEARCH_USERNAME=kibana_system
- ELASTICSEARCH_PASSWORD=<KIBANA_SYSTEM_PASSWORD>
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
mem_limit: 1073741824
healthcheck:
test:
[
"CMD-SHELL",
"curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
]
interval: 10s
timeout: 10s
retries: 120
I've successfully setup the Container Scanning feature from GitLab for a single Docker image. Now I'd like to scan yet another image using the same CI/CD configuration in .gitlab-ci.yml
Problem
It looks like it is not possible to have multiple Container Scanning reports on the Merge Request detail page.
The following screenshot shows the result of both Container Scanning jobs in the configuration below.
We scan two Docker images, which both have CVE's to be reported:
iojs:1.6.3-slim (355 vulnerabilities)
golang:1.3 (1139 vulnerabilities)
Expected result
The Container Scanning report would show a total of 1494 vulnerabilities (355 + 1139). Currently it looks like only the results for the golang image are being included.
Relevant parts of the configuration
container_scanning_first_image:
script:
- docker pull golang:1.3
- ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report-first-image.json -l clair.log golang:1.3 || true
artifacts:
reports:
container_scanning: gl-container-scanning-report-first-image.json
container_scanning_second_image:
script:
- docker pull iojs:1.6.3-slim
- ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report-second-image.json -l clair.log iojs:1.6.3-slim || true
artifacts:
reports:
container_scanning: gl-container-scanning-report-second-image.json
Full configuration for reference
image: docker:stable
stages:
- scan
variables:
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
container_scanning_first_image:
stage: scan
variables:
GIT_STRATEGY: none
DOCKER_SERVICE: docker
DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/
CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1
NO_PROXY: ${DOCKER_SERVICE},localhost
allow_failure: true
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker run -d --name db arminc/clair-db:latest
- docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
- apk add -U wget ca-certificates
- docker pull golang:1.3
- wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
- mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner
- touch clair-whitelist.yml
- retries=0
- echo "Waiting for clair daemon to start"
- while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
- ./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report-first-image.json -l clair.log golang:1.3 || true
artifacts:
paths:
- gl-container-scanning-report-first-image.json
reports:
container_scanning: gl-container-scanning-report-first-image.json
dependencies: []
only:
refs:
- branches
variables:
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
except:
variables:
- $CONTAINER_SCANNING_DISABLED
container_scanning_second_image:
stage: scan
variables:
GIT_STRATEGY: none
DOCKER_SERVICE: docker
DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/
CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1
NO_PROXY: ${DOCKER_SERVICE},localhost
allow_failure: true
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker run -d --name db arminc/clair-db:latest
- docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
- apk add -U wget ca-certificates
- docker pull iojs:1.6.3-slim
- wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
- mv clair-scanner_linux_amd64 clair-scanner
- chmod +x clair-scanner
- touch clair-whitelist.yml
- retries=0
- echo "Waiting for clair daemon to start"
- while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
- ./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report-second-image.json -l clair.log iojs:1.6.3-slim || true
artifacts:
paths:
- gl-container-scanning-report-second-image.json
reports:
container_scanning: gl-container-scanning-report-second-image.json
dependencies: []
only:
refs:
- branches
variables:
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
except:
variables:
- $CONTAINER_SCANNING_DISABLED
Question
How should the GitLab Container Scanning feature be configured in order to be able to report the results of two Docker images?
I've configured Zookeeper and Kafka containers in a fig.yml file for Docker. Both containers start fine. But after sending a number of messages, my application /zk-client hangs. On checking zookeeper logs, I see the error:
Error Path:/brokers Error:KeeperErrorCode = NoNode for /brokers
My fig.yml is as follows:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
environment:
ZK_ADVERTISED_HOST_NAME: xx.xx.x.xxx
ZK_CONNECTION_TIMEOUT_MS: 6000
ZK_SYNC_TIME_MS: 2000
ZK_DATADIR: /path/to/data/zk/data/dir
kafka:
image: wurstmeister/kafka:0.8.2.0
ports:
- "xx.xx.x.xxx:9092:9092"
links:
- zookeeper:zk
environment:
KAFKA_ADVERTISED_HOST_NAME: xx.xx.x.xxx
KAFKA_LOG_DIRS: /home/svc_cis4/dl
volumes:
- /var/run/docker.sock:/var/run/docker.sock
I've searched for quite a while now, but I haven't got a solution yet. I've also tried setting the data directory in fig.yml using ZK_DATADIR: '/path/to/zk/data/dir' but it doesn't seem to help. Any assistance will be appreciated.
UPDATE
Content of /opt/kafka_2.10-0.8.2.0/config/server.properties:
broker.id=0
port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=1
num.recovery.threads.per.data.dir=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
log.cleaner.enable=false
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=6000
The problems you are having are not related with zookeeper's data directory. The error Error Path:/brokers Error:KeeperErrorCode = NoNode for /brokers are due to your application cannot find any broker znode in zookeeper's data. This is happening probably because the kafka container is not connecting correctly with zookeeper, and looking to wurstmeister's images I think the problem may be related to variable KAFKA_ADVERTISED_HOST_NAME could be wrong. I don't know if there is a reason to assign that variable through a env variable that has to be passed, but from my point of view this is not a good approach. There are multiple ways to configure kafka (in fact there is no need to set advertised.host.name and you can leave it commented and kafka will take default hostname, which can be set with docker), but a fast solution using this would be editing start-kafka.sh and rebuilding the image:
#!/bin/bash
if [[ -z "$KAFKA_ADVERTISED_PORT" ]]; then
export KAFKA_ADVERTISED_PORT=$(docker port `hostname` 9092 | sed -r "s/.*:(.*)/\1/g")
fi
if [[ -z "$KAFKA_BROKER_ID" ]]; then
export KAFKA_BROKER_ID=$KAFKA_ADVERTISED_PORT
fi
if [[ -z "$KAFKA_LOG_DIRS" ]]; then
export KAFKA_LOG_DIRS="/kafka/kafka-logs-$KAFKA_BROKER_ID"
fi
if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
export KAFKA_ZOOKEEPER_CONNECT=$(env | grep ZK.*PORT_2181_TCP= | sed -e 's|.*tcp://||' | paste -sd ,)
fi
if [[ -n "$KAFKA_HEAP_OPTS" ]]; then
sed -r -i "s/^(export KAFKA_HEAP_OPTS)=\"(.*)\"/\1=\"$KAFKA_HEAP_OPTS\"/g" $KAFKA_HOME/bin/kafka-server-start.sh
unset KAFKA_HEAP_OPTS
fi
for VAR in `env`
do
if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_HOME ]]; then
kafka_name=`echo "$VAR" | sed -r "s/KAFKA_(.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .`
env_var=`echo "$VAR" | sed -r "s/(.*)=.*/\1/g"`
if egrep -q "(^|^#)$kafka_name=" $KAFKA_HOME/config/server.properties; then
sed -r -i "s#(^|^#)($kafka_name)=(.*)#\2=${!env_var}#g" $KAFKA_HOME/config/server.properties #note that no config values may contain an '#' char
else
echo "$kafka_name=${!env_var}" >> $KAFKA_HOME/config/server.properties
fi
fi
done
###NEW###
IP=$(hostname --ip-address)
sed -i -e "s/^advertised.host.name.*/advertised.host.name=$IP/" $KAFKA_HOME/config/server.properties
###END###
$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties
If this doesn't solve your problem you can get more information starting a session inside the containers (i.e.: docker exec -it kafkadocker_kafka_1 /bin/bash for kafka's and docker exec -it kafkadocker_zookeeper_1 /bin/bash for zookeeper's), and there check kafka logs, or zookeeper console (/opt/zookeeper-3.4.6/bin/zkCli.sh)
The configuration that's been working for me without any issues for the last two days involves specifying host addresses for both Zookeeper and Kafka. My fig.yml content is:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "xx.xx.x.xxx:2181:2181"
kafka:
image: wurstmeister/kafka:0.8.2.0
ports:
- "9092:9092"
links:
- zookeeper:zk
environment:
KAFKA_ADVERTISED_HOST_NAME: xx.xx.x.xxx
KAFKA_NUM_REPLICA_FETCHERS: 4
...other env variables...
volumes:
- /var/run/docker.sock:/var/run/docker.sock
validator:
build: .
volumes:
- .:/host
entrypoint: /bin/bash
command: -c 'java -jar /host/app1.jar'
links:
- zookeeper:zk
- kafka
analytics:
build: .
volumes:
- .:/host
entrypoint: /bin/bash
command: -c 'java -jar /host/app2.jar'
links:
- zookeeper:zk
- kafka
loader:
build: .
volumes:
- .:/host
entrypoint: /bin/bash
command: -c 'java -jar /host/app3.jar'
links:
- zookeeper:zk
- kafka
And the accompanying Dockerfile content:
FROM ubuntu:trusty
MAINTAINER Wurstmeister
RUN apt-get update; apt-get install -y unzip openjdk-7-jdk wget git docker.io
RUN wget -q http://apache.mirrors.lucidnetworks.net/kafka/0.8.2.0/kafka_2.10-0.8.2.0.tgz -O /tmp/kafka_2.10-0.8.2.0.tgz
RUN tar xfz /tmp/kafka_2.10-0.8.2.0.tgz -C /opt
VOLUME ["/kafka"]
ENV KAFKA_HOME /opt/kafka_2.10-0.8.2.0
ADD start-kafka.sh /usr/bin/start-kafka.sh
ADD broker-list.sh /usr/bin/broker-list.sh
CMD start-kafka.sh