How to prevent td-agent / fluentd from inserting undesirable metadata? - fluentd

Question

Please assist me in understanding how to prevent td-agent from inserting undesirable metadata.
It transforms a record of the form JSONBLOB to TIMESTAMP LOGNAME JSONBLOB.
I Only want the json, not the timestamp and logname.
for example --
td-agent transforms a log that looks like this:
{"log":"I0123 01:58:21.668297 1 nanny.go:108] dnsmasq[14]: 130404 192.168.178.209/44096 reply bitesize-docker-registry.s3.amazonaws.com.cluster.local is NXDOMAIN\n","stream":"stderr","docker":{"container_id":"52d1f2122ea4d144fb07835e1d8b7d210e2ac05c4c0bfd7d2e09237b597bf6a3"},"kubernetes":{"container_name":"dnsmasq","namespace_name":"kube-system","pod_name":"kube-dns-5c9464f66b-whljm"},"target_index":"kube-system-1970.01.01"}
to this:
1970-01-01T00:33:40+00:00 kubernetes.var.log.containers.kube-dns-5c9464f66b-whljm_kube-system_dnsmasq-52d1f2122ea4d144fb07835e1d8b7d210e2ac05c4c0bfd7d2e09237b597bf6a3.log {"log":"I0123 01:58:21.668297 1 nanny.go:108] dnsmasq[14]: 130404 192.168.178.209/44096 reply bitesize-docker-registry.s3.amazonaws.com.cluster.local is NXDOMAIN\n","stream":"stderr","docker":{"container_id":"52d1f2122ea4d144fb07835e1d8b7d210e2ac05c4c0bfd7d2e09237b597bf6a3"},"kubernetes":{"container_name":"dnsmasq","namespace_name":"kube-system","pod_name":"kube-dns-5c9464f66b-whljm"},"target_index":"kube-system-1970.01.01"}
my config looks like this:
<source>
#type tail
#id in_tail_container_logs
path /var/log/containers/*.log
exclude_path ["/var/log/containers/td-agent*.log"]
pos_file /var/log/td-agent/td-agent-containers.log.pos
tag kubernetes.*
read_from_head true
keep_time_key true
<parse>
#type json
json_parser json
time_format %Y-%m-%dT%H:%M:%S.%NZ
</parse>
</source>
<filter kubernetes.var.log.containers.**>
#type parser
<parse>
#type json
json_parser json
</parse>
replace_invalid_sequence true
emit_invalid_record_to_error false
key_name log
reserve_data true
</filter>
#Filter only kubernetes logs
<filter kubernetes.**>
#type grep
regexp kubernetes namespace_name
</filter>
<filter **>
#type record_transformer
enable_ruby
<record>
target_index system_log
</record>
</filter>
<filter kubernetes.**>
#type record_transformer
enable_ruby
<record>
target_index ${record["kubernetes"]["namespace_name"]}-${time.strftime('%Y.%m.%d')}
</record>
</filter>
<match fluent.**>
#type null
</match>
# relabel
<match kubernetes.**>
#type copy
<store>
#type relabel
#label #AWSES
</store>
<store>
#type relabel
#label #CCL
</store>
</match>
# AWS ElasticSearch logging
<label #AWSES>
<match kubernetes.**>
#type aws-elasticsearch-service
#log_level info
ssl_verify false
reload_connections false
time_key "time"
ssl_version TLSv1_2
resurrect_after 5s
target_index_key target_index
logstash_format true
include_tag_key false
type_name "access_log"
<buffer>
flush_mode interval
retry_type exponential_backoff
flush_thread_count 4
flush_interval 5s
retry_forever true
retry_max_interval 30
chunk_limit_size 32M
queue_limit_length 32
total_limit_size 5G
queued_chunks_limit_size 100
overflow_action block
disable_chunk_backup true
</buffer>
<endpoint>
url https://elasticsearch.idoug0122.us-east-2.dev:443
region us-east-2
</endpoint>
</match>
</label>
# CISO Centralized logging
<label #CCL>
<match kubernetes.**>
#type s3
s3_bucket loggingbucket-us-east-2-602604727914
s3_region us-east-2
path k8s/dev/idoug0122/
<buffer>
flush_mode interval
retry_type exponential_backoff
flush_thread_count 4
flush_interval 5s
retry_forever true
retry_max_interval 30
chunk_limit_size 32M
queue_limit_length 32
total_limit_size 5G
queued_chunks_limit_size 100
overflow_action block
disable_chunk_backup true
</buffer>
</match>
</label>
Any help would be appreciated!

Answer 1

the desirable format is obtained by adding a format directive under either a 'filter' or 'match' section:
<match kubernetes.**>
<format>
#type json
</format>
</match kubernetes.**>