Fluentd logging driver sends unstructured log message - fluentd

Question

My environment has a setup where docker container logs are forwarded to fluentd, then fluentd forwards to splunk.
I have a issue with fluentd, some of the docker container logs are not in structured format. From the documentation i see that:
fluentd log driver sends the following metadata in the structured log message:
container_id,
container_name,
source,
log
My issue is few of the logs have unstructured metadata information:
for example:
Log 1:
{"log":"2019/03/12 13:59:49 [info] 6#6: *2425596 client closed connection while waiting for request, client: 10.17.84.12, server: 0.0.0.0:80","container_id":"789459f8f8a52c8b4b","container_name":"testingcontainer-1ed-fwij4-EcsTaskDefinition-1TF1DH,"source":"stderr"}
Log 2:
{"container_id":"26749a26500dd04e92fc","container_name":"/4C4DTHQR2V6C-EcsTaskDefinition-1908NOZPKPKY0-1","source":"stdout","log":"\u001B[0mGET \u001B[32m200 \u001B[0m0.634 ms - -\u001B[0m"}
These two logs have different order of metadata information(log1-[log, conatiner-name, container_id, source])(log2- [container_id, conatiner-name, source, log]). Because of this i'm getting some issues in splunk. How can i resolve this to get same order of metadata info?
my fluend config file is
<source>
#type forward
#id input1
#label #mainstream
#log_level trace
port 24224
</source>
<label #mainstream>
<match *.**>
type copy
<store>
#type file
#id output_docker1
path /fluentd/log/docker.*.log
symlink_path /fluentd/log/docker.log
append true
time_slice_format %Y%m%d
time_slice_wait 1m
time_format %Y%m%dT%H%M%S%z
utc
buffer_chunk_limit 512m
</store>
<store>
#type s3
#id output_docker2
#log_level trace
s3_bucket bucketwert-1
s3_region us-east-1
path logs/
buffer_path /fluentd/log/docker.log
s3_object_key_format %{path}%{time_slice}_sbx_docker_%{index}.%{file_extension}
flush_interval 3600s
time_slice_format %Y%m%d
time_format %Y%m%dT%H%M%S%z
utc
buffer_chunk_limit 512m
</store>
</match>
</label>

Answer 1

How about fluent-plugin-record-sort?
Or you can use built-in record_trandformer plugin like the following if you know all keys in a record:
<source>
#type dummy
tag dummy
dummy [
{"log": "log1", "container_id": "123", "container_name": "name1", "source": "stderr"},
{"container_id": "456", "container_name": "name2", "source": "stderr", "log": "log2"}
]
</source>
<filter dummy>
#type record_transformer
renew_record true
keep_keys log,container_id,container_name,source
</filter>
<match dummy>
#type stdout
</match>
UPDATE(not tested):
<source>
#type forward
#id input1
#label #mainstream
#log_level trace
port 24224
</source>
<label #mainstream>
<filter>
#type record_transformer
renew_record true
keep_keys log,container_id,container_name,source
</filter>
<match *.**>
#type copy
<store>
#type file
#id output_docker1
path /fluentd/log/docker.*.log
symlink_path /fluentd/log/docker.log
append true
time_slice_format %Y%m%d
time_slice_wait 1m
time_format %Y%m%dT%H%M%S%z
utc
buffer_chunk_limit 512m
</store>
<store>
#type s3
#id output_docker2
#log_level trace
s3_bucket bucketwert-1
s3_region us-east-1
path logs/
buffer_path /fluentd/log/docker.log
s3_object_key_format %{path}%{time_slice}_sbx_docker_%{index}.%{file_extension}
flush_interval 3600s
time_slice_format %Y%m%d
time_format %Y%m%dT%H%M%S%z
utc
buffer_chunk_limit 512m
</store>
</match>
</label>