Transcoder fails to write output due to permission issue - google-cloud-transcoder

The transcode job is able to read the file but the output directory is not getting created.
I checked the log and the error is due to permission issue with Storage. I would like to get your help related to this issue.
Logs:
The response of the video transcoding is as follows.
{
"name": "projects/PROJECT-NUMBER/locations/us-central1/jobs/JOB-ID",
"config": {
"inputs": [
{
"key": "input0",
"uri": "gs://input.mp4"
}
],
"editList": [
{
"key": "atom0",
"inputs": [
"input0"
],
"startTimeOffset": "0s"
}
],
"elementaryStreams": [
{
"videoStream": {
"codec": "h264",
"profile": "high",
"preset": "veryfast",
"heightPixels": 720,
"widthPixels": 1280,
"pixelFormat": "yuv420p",
"bitrateBps": 1500000,
"rateControlMode": "vbr",
"crfLevel": 21,
"vbvSizeBits": 1500000,
"vbvFullnessBits": 1350000,
"gopDuration": "3s",
"entropyCoder": "cabac",
"frameRate": 60
},
"key": "video-streamR480P"
},
{
"videoStream": {
"codec": "h264",
"profile": "high",
"preset": "veryfast",
"heightPixels": 1080,
"widthPixels": 1920,
"pixelFormat": "yuv420p",
"bitrateBps": 3000000,
"rateControlMode": "vbr",
"crfLevel": 21,
"vbvSizeBits": 3000000,
"vbvFullnessBits": 2700000,
"gopDuration": "3s",
"entropyCoder": "cabac",
"frameRate": 60
},
"key": "video-streamR720P"
},
{
"videoStream": {
"codec": "h264",
"profile": "high",
"preset": "veryfast",
"heightPixels": 1440,
"widthPixels": 2560,
"pixelFormat": "yuv420p",
"bitrateBps": 6000000,
"rateControlMode": "vbr",
"crfLevel": 21,
"vbvSizeBits": 6000000,
"vbvFullnessBits": 5400000,
"gopDuration": "3s",
"entropyCoder": "cabac",
"frameRate": 60
},
"key": "video-streamR1080P"
},
{
"videoStream": {
"codec": "h264",
"profile": "high",
"preset": "veryfast",
"heightPixels": 2160,
"widthPixels": 3840,
"pixelFormat": "yuv420p",
"bitrateBps": 10000000,
"rateControlMode": "vbr",
"crfLevel": 21,
"vbvSizeBits": 10000000,
"vbvFullnessBits": 9000000,
"gopDuration": "3s",
"entropyCoder": "cabac",
"frameRate": 60
},
"key": "video-streamR1440P"
},
{
"audioStream": {
"codec": "aac",
"bitrateBps": 512000,
"channelCount": 2,
"channelLayout": [
"fl",
"fr"
],
"sampleRateHertz": 48000
},
"key": "audio-stream"
}
],
"muxStreams": [
{
"key": "mux-streamR480P",
"fileName": "mux-streamR480P.m4s",
"container": "fmp4",
"elementaryStreams": [
"video-streamR480P"
]
},
{
"key": "mux-streamR720P",
"fileName": "mux-streamR720P.m4s",
"container": "fmp4",
"elementaryStreams": [
"video-streamR720P"
]
},
{
"key": "mux-streamR1080P",
"fileName": "mux-streamR1080P.m4s",
"container": "fmp4",
"elementaryStreams": [
"video-streamR1080P"
]
},
{
"key": "mux-streamR1440P",
"fileName": "mux-streamR1440P.m4s",
"container": "fmp4",
"elementaryStreams": [
"video-streamR1440P"
]
},
{
"key": "mux-streamaudio-stream",
"fileName": "mux-streamaudio-stream.m4s",
"container": "fmp4",
"elementaryStreams": [
"audio-stream"
]
}
],
"manifests": [
{
"fileName": "manifest.mpd",
"type": "DASH",
"muxStreams": [
"mux-streamR480P",
"mux-streamR720P",
"mux-streamR1080P",
"mux-streamR1440P",
"mux-streamaudio-stream"
]
},
{
"fileName": "master.m3u8",
"type": "HLS",
"muxStreams": [
"mux-streamR480P",
"mux-streamR720P",
"mux-streamR1080P",
"mux-streamR1440P",
"mux-streamaudio-stream"
]
}
],
"output": {
"uri": "gs://output-bucket/"
}
}, "ttlAfterCompletionDays": 30
}
Job_ID is XXXXXXXXXXXXXXXXXXXXXXXX.
Please help.

I have checked the job and found the failure message:
Job validation failed: atom atom0 does not have any inputs (input0) with an audio track
It has nothing to do with storage permission. Can you fix the edit list with an input that includes audio track and try again?
I think the permission error that you observe is happening on the client side and is not related to the Transcoder API

Related

How to get a bone's weight using a json file

I am trying to get the weight for every bone using a gltf json file but i dont know how
here is the json file i am trying to get the weight from:
{
"accessors": [
{
"bufferView": 0,
"componentType": 5126,
"count": 317,
"max": [
10.570027351379395,
15.918471336364746,
6.125584125518799
],
"min": [
-2.611180305480957,
-1.9711413383483887,
-7.055622577667236
],
"type": "VEC3"
},
{
"bufferView": 1,
"componentType": 5126,
"count": 317,
"type": "VEC3"
},
{
"bufferView": 2,
"componentType": 5126,
"count": 317,
"type": "VEC2"
},
{
"bufferView": 3,
"componentType": 5121,
"count": 317,
"type": "VEC4"
},
{
"bufferView": 4,
"componentType": 5126,
"count": 317,
"type": "VEC4"
},
{
"bufferView": 5,
"componentType": 5123,
"count": 768,
"type": "SCALAR"
},
{
"bufferView": 6,
"componentType": 5126,
"count": 5,
"type": "MAT4"
}
],
"asset": {
"generator": "Khronos glTF Blender I/O v1.5.17",
"version": "2.0"
},
"bufferViews": [
{
"buffer": 0,
"byteLength": 3804,
"byteOffset": 0
},
{
"buffer": 0,
"byteLength": 3804,
"byteOffset": 3804
},
{
"buffer": 0,
"byteLength": 2536,
"byteOffset": 7608
},
{
"buffer": 0,
"byteLength": 1268,
"byteOffset": 10144
},
{
"buffer": 0,
"byteLength": 5072,
"byteOffset": 11412
},
{
"buffer": 0,
"byteLength": 1536,
"byteOffset": 16484
},
{
"buffer": 0,
"byteLength": 320,
"byteOffset": 18020
}
],
"buffers": [
{
"byteLength": 18340,
"uri": "manoe.bin"
}
],
"images": [
{
"mimeType": "image/png",
"name": "Material_002_baseColor",
"uri": "Material_002_baseColor.png"
}
],
"materials": [
{
"doubleSided": true,
"name": "Material.001",
"pbrMetallicRoughness": {
"baseColorTexture": {
"index": 0
},
"metallicFactor": 0,
"roughnessFactor": 0.4000000059604645
}
}
],
"meshes": [
{
"name": "Cone.001",
"primitives": [
{
"attributes": {
"JOINTS_0": 3,
"NORMAL": 1,
"POSITION": 0,
"TEXCOORD_0": 2,
"WEIGHTS_0": 4
},
"indices": 5,
"material": 0
}
]
}
],
"nodes": [
{
"name": "Bone.004",
"rotation": [
0.17750653624534607,
-2.8648443617385055e-8,
-0.9102426171302795,
0.37409862875938416
],
"scale": [
1.000000238418579,
1,
1
],
"translation": [
-4.411018750261064e-8,
0.8281745314598083,
1.4894996525072202e-7
]
},
{
"children": [
0
],
"name": "Bone.002",
"rotation": [
0.2184765785932541,
-0.2617807984352112,
-0.7560504078865051,
0.558682918548584
],
"scale": [
1.0000003576278687,
1,
1
],
"translation": [
6.093483762015239e-8,
0.8281747698783875,
-1.283853023892334e-9
]
},
{
"name": "Bone.003",
"rotation": [
-0.4916436970233917,
7.804754176277129e-8,
-0.14189021289348602,
0.8591586947441101
],
"scale": [
1,
1.0000001192092896,
0.9999999403953552
],
"translation": [
-2.4224684125329077e-7,
0.8281750679016113,
-2.68131650216219e-9
]
},
{
"children": [
1,
2
],
"name": "Bone.001",
"rotation": [
-0.11594496667385101,
0.08448944985866547,
-0.5036448836326599,
0.8519155383110046
],
"scale": [
1.000000238418579,
0.9999995827674866,
1
],
"translation": [
-1.357730283757519e-8,
1.0000007152557373,
-3.471412068391244e-14
]
},
{
"children": [
3
],
"name": "Bone",
"rotation": [
2.3709270635663415e-7,
2.8494374859633353e-8,
6.9593789722201e-15,
1
],
"scale": null,
"translation": null
},
{
"mesh": 0,
"name": "Cone.001",
"skin": 0
},
{
"children": [
5,
4
],
"name": "ArmatureBase",
"scale": [
5.400000095367432,
5.400000095367432,
5.400000095367432
]
}
],
"samplers": [
{
"magFilter": 9729,
"minFilter": 9987
}
],
"scene": 0,
"scenes": [
{
"name": "Scene",
"nodes": [
6
]
}
],
"skins": [
{
"inverseBindMatrices": 6,
"joints": [
4,
3,
1,
0,
2
],
"name": "ArmatureBase"
}
],
"textures": [
{
"sampler": 0,
"source": 0
}
]
}
The glTF overview card summarizes it nicely:
Every vertex can be associated with up to 4 joints, each with a given weight.
These are encoded in the components of the JOINTS_0 and WEIGHTS_0 accessors. In your case those are accessors 3 and 4 and bufferviews 3 and 4 with a component type of byte and float, respectively.
One more indirection into the bufferViews component here tells you where in the binary file to look.

Tweepy - Differences with status info delivered by Twitter API

Does somebody knows why Twitter API delivers different information for the same status using different methods?
I'm working on a Twitter bot to collect transit events. I'm facing some issues with the information delivered by the API, due to I've identified that depending on the method, the API can deliver different result for the same Tweet/status.
Here an example for the Tweet ID:1531517617562951682 using api.get_status, the status doesn't have the attributes extended_tweet and timestamp_ms.
Thanks in advance.
{
"created_at": "Tue May 31 06:07:08 +0000 2022",
"id": 1531517617562951682,
"id_str": "**1531517617562951682**",
"text": "01:07 #Precauci\u00f3nVial | Por percance en Av. Ing. Eduardo Molina a la altura de Av. Talism\u00e1n, col. San Pedro el Chic\u2026 (**removed_url)",
**"truncated": true**,
"entities": {
"hashtags": [
{
"text": "Precauci\u00f3nVial",
"indices": [
6,
21
]
}
],
"symbols": [],
"user_mentions": [],
"urls": [
{
"url": **removed_url_short,
"expanded_url": ,
"display_url": ,
"indices": [
117,
140
]
}
]
},
"source": "Twitter Web App",
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 166594238,
"id_str": "166594238",
"name": "OVIAL_SSCCDMX",
"screen_name": "OVIALCDMX",
"location": "Liverpool #136, Col. Ju\u00e1rez",
"description": "Centro de Orientaci\u00f3n Vial de la Secretar\u00eda de Seguridad Ciudadana de la Ciudad de M\u00e9xico.",
"url":,
"entities": {
"url": {
"urls": [
{
"url": ,
"expanded_url": ,
"display_url": "ssc.cdmx.gob.mx",
"indices": [
0,
23
]
}
]
},
"description": {
"urls": []
}
},
"protected": false,
"followers_count": 1866313,
"friends_count": 125,
"listed_count": 4188,
"created_at": "Wed Jul 14 15:00:43 +0000 2010",
"favourites_count": 2748,
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"verified": true,
"statuses_count": 1110675,
"lang": null,
"contributors_enabled": false,
"is_translator": false,
"is_translation_enabled": false,
"profile_background_color": "000000",
"profile_background_image_url":
"profile_background_image_url_https": ,
"profile_background_tile": false,
"profile_image_url": ,
"profile_image_url_https": ,
"profile_banner_url": ,
"profile_link_color": "BC8F8F",
"profile_sidebar_border_color": "000000",
"profile_sidebar_fill_color": "000000",
"profile_text_color": "000000",
"profile_use_background_image": false,
"has_extended_profile": false,
"default_profile": false,
"default_profile_image": false,
"following": false,
"follow_request_sent": false,
"notifications": false,
"translator_type": "none",
"withheld_in_countries": []
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"is_quote_status": false,
"retweet_count": 0,
"favorite_count": 0,
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"possibly_sensitive_appealable": false,
"lang": "es"
}
Same status collected with a stream.Attributes extended_tweet and timestamp are present on the status:
{
"created_at": "Tue May 31 06:07:08 +0000 2022",
"id": 1531517617562951682,
"id_str": "1531517617562951682",
"text": "01:07 #Precauci\u00f3nVial | Por percance en Av. Ing. Eduardo Molina a la altura de Av. Talism\u00e1n, col. San Pedro el Chic\u2026 (**removed url)",
"display_text_range": [
0,
140
],
"source": "Twitter Web App",
"truncated": true,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 166594238,
"id_str": "166594238",
"name": "OVIAL_SSCCDMX",
"screen_name": "OVIALCDMX",
"location": "Liverpool #136, Col. Ju\u00e1rez",
"url": "http://ssc.cdmx.gob.mx",
"description": "Centro de Orientaci\u00f3n Vial de la Secretar\u00eda de Seguridad Ciudadana de la Ciudad de M\u00e9xico.",
"translator_type": "none",
"protected": false,
"verified": true,
"followers_count": 1866203,
"friends_count": 125,
"listed_count": 4188,
"favourites_count": 2748,
"statuses_count": 1110325,
"created_at": "Wed Jul 14 15:00:43 +0000 2010",
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"lang": null,
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "000000",
"profile_background_image_url": ,
"profile_background_image_url_https": ,
"profile_background_tile": false,
"profile_link_color": "BC8F8F",
"profile_sidebar_border_color": "000000",
"profile_sidebar_fill_color": "000000",
"profile_text_color": "000000",
"profile_use_background_image": false,
"profile_image_url": ,
"profile_image_url_https": ,
"profile_banner_url": ,
"default_profile": false,
"default_profile_image": false,
"following": null,
"follow_request_sent": null,
"notifications": null,
"withheld_in_countries": []
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"is_quote_status": false,
"extended_tweet": {
"full_text": "01:07 #Precauci\u00f3nVial | Por percance en Av. Ing. Eduardo Molina a la altura de Av. Talism\u00e1n, col. San Pedro el Chico, servicios de emergencia acuden al lugar. (**removed url)",
"display_text_range": [
0,
158
],
"entities": {
"hashtags": [
{
"text": "Precauci\u00f3nVial",
"indices": [
6,
21
]
}
],
"urls": [],
"user_mentions": [],
"symbols": [],
"media": [
{
"id": 1531517217703092224,
"id_str": "1531517217703092224",
"indices": [
159,
182
],
"additional_media_info": {
"monetizable": false
},
"media_url": ,
"media_url_https": ,
"url":,
"display_url": "pic.twitter.com/q3seKvyPIM",
"expanded_url": ,
"type": "video",
"video_info": {
"aspect_ratio": [
16,
9
],
"duration_millis": 4003,
"variants": [
{
"content_type": "application/x-mpegURL",
"url": ""
},
{
"bitrate": 832000,
"content_type": "video/mp4",
"url": "
},
{
"bitrate": 256000,
"content_type": "video/mp4",
"url":
}
]
},
"sizes": {
"thumb": {
"w": 150,
"h": 150,
"resize": "crop"
},
"large": {
"w": 640,
"h": 360,
"resize": "fit"
},
"medium": {
"w": 640,
"h": 360,
"resize": "fit"
},
"small": {
"w": 640,
"h": 360,
"resize": "fit"
}
}
}
]
},
"extended_entities": {
"media": [
{
"id": 1531517217703092224,
"id_str": "1531517217703092224",
"indices": [
159,
182
],
"additional_media_info": {
"monetizable": false
},
"media_url": ,
"media_url_https": ,
"url": ,
"display_url": ,
"expanded_url": ,
"type": "video",
"video_info": {
"aspect_ratio": [
16,
9
],
"duration_millis": 4003,
"variants": [
{
"content_type": "application/x-mpegURL",
"url": "
},
{
"bitrate": 832000,
"content_type": "video/mp4",
"url": "
},
{
"bitrate": 256000,
"content_type": "video/mp4",
"url": ""
}
]
},
"sizes": {
"thumb": {
"w": 150,
"h": 150,
"resize": "crop"
},
"large": {
"w": 640,
"h": 360,
"resize": "fit"
},
"medium": {
"w": 640,
"h": 360,
"resize": "fit"
},
"small": {
"w": 640,
"h": 360,
"resize": "fit"
}
}
}
]
}
},
"quote_count": 0,
"reply_count": 0,
"retweet_count": 0,
"favorite_count": 0,
"entities": {
"hashtags": [
{
"text": "Precauci\u00f3nVial",
"indices": [
6,
21
]
}
],
"urls": [
{
"url": ,
"expanded_url": ,
"display_url": "twitter.com/i/web/status/1\u2026",
"indices": [
117,
140
]
}
],
"user_mentions": [],
"symbols": []
},
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"filter_level": "low",
"lang": "es",
"**timestamp_ms**": "1653977228905"
}
For v1.1, the extended Tweet information is available by default in the streaming API, but for the REST endpoints, you need to add the tweet_mode=extended parameter to your request to retrieve the full text.
In V2, there is no longer a concept of extended (beyond 140 characters) Tweets. All Tweets are natively up to 280 characters as standard.

Individual transcode times 2-3X that VS all done in single job/config file in google cloud

Transcode jobs taking exponentially longer if done individually vs as part of a single config file/job.
First, use a config file to transcode 5 or more versions of a piece of content, note the total transcode time. Second, break out each version from the original config file into single transcode jobs and note the time each takes individually, add them all up then compare to the time it took to do the first job when done as a whole/single config file. I am seeing a 2-3X increase in transcode times.
Job Config { "config": { "elementaryStreams": [ { "key": "360p1-125kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 640, "heightPixels": 360, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 125000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "360p2-250kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 640, "heightPixels": 360, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 250000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "480p1-400kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 854, "heightPixels": 480, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 400000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "480p2-800kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 854, "heightPixels": 480, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 800000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "720p-1600kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 1280, "heightPixels": 720, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 1600000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "720p-2500kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 1280, "heightPixels": 720, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 2500000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } }, { "key": "1080p-5500kbps-h264", "videoStream": { "codec": "h264", "profile": "high", "preset": "slow", "widthPixels": 1920, "heightPixels": 1080, "frameRate": 29.97, "pixelFormat": "yuv420p", "bitrateBps": 5500000, "rateControlMode": "vbr", "enableTwoPass": true, "gopDuration": "4s", "allowOpenGop": true, "entropyCoder": "cabac", "bFrameCount": 3, "bPyramid": true } } ], "muxStreams": [ { "key": "360p1-125kbps-h264", "container": "mp4", "elementaryStreams": [ "360p1-125kbps-h264" ] }, { "key": "360p2-250kbps-h264", "container": "mp4", "elementaryStreams": [ "360p2-250kbps-h264" ] }, { "key": "460p1-400kbps-h264", "container": "mp4", "elementaryStreams": [ "480p1-400kbps-h264" ] }, { "key": "460p2-800kbps-h264", "container": "mp4", "elementaryStreams": [ "480p2-800kbps-h264" ] }, { "key": "720p-1600kbps-h264", "container": "mp4", "elementaryStreams": [ "720p-1600kbps-h264" ] }, { "key": "720p-2500kbps-h264", "container": "mp4", "elementaryStreams": [ "720p-2500kbps-h264" ] }, { "key": "1080p-5500kbps-h264", "container": "mp4", "elementaryStreams": [ "1080p-5500kbps-h264" ] } ] } }
This behavior is working as intended mainly because there's a lot of duplicate work done with many jobs (so things like downloading the input file and other parts of our pipeline need to happen many times rather than once), and scheduling transcodes internally for many jobs will most likely never be as fast as well.
Transcoder API handles transcoding in parallel (we call it partition processing. Divide the long input file into partitions, and process each partition(2 min long video by default) in parallel) no matter your config is with 1 elementary stream or 10 elementary streams.
Back to your original question: 1 job with 10 elementary streams v.s. 10 jobs each with 1 elementary stream.
They all handled in partition processing(parallel transcoding), however, 10 jobs each with 1 elementary stream do lots of duplicated tasks each job, thus requires longer time to finish in the end if you simply add the time together.

How to add dashboard configuration json file in Grafana image?

I have a grafana docker image which have hawkular-datasource pre-configured using configuration files.
After after running grafana instance, I have a json given by teammate, which can be imported inside grafana and that json file creates dashboard when imported.
How do I make that dashboards appear by default in Grafana instance?
I tried copying the json file to /etc/grafana/provisioning/dashboards/ folder and created a new docker image. But when I run the image, the instance doesn't contain the dashboard at the homepage or anywhere in it.
How do I add this json file in docker image. Am I following the correct way?
I tried this http://docs.grafana.org/administration/provisioning/ But it didn't help out much. Any suggestion?
Here is the json file.
{
"id": null,
"title": "Openshift Metrics",
"tags": [],
"style": "dark",
"timezone": "browser",
"editable": true,
"hideControls": false,
"sharedCrosshair": false,
"rows": [
{
"collapse": false,
"editable": true,
"height": "322px",
"panels": [
{
"content": "<center><p style='font-size: 40pt'>$app</p></center>",
"editable": true,
"error": false,
"id": 23,
"isNew": true,
"links": [],
"mode": "html",
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"style": {
"font-size": "36pt"
},
"title": "",
"type": "text"
},
{
"aliasColors": {},
"bars": false,
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"fill": 1,
"grid": {
"threshold1": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2": null,
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"id": 9,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeatIteration": 1476706310439,
"scopedVars": {},
"seriesOverrides": [],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "none",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "avg",
"type": "gauge"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Memory usage",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"show": true
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 12,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "sum",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "live",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Live, all pods",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 15,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "avg",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "live",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Live per pod",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 10,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "sum",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "avg",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Average, all pods",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 13,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "avg",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "avg",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Average per pod",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 11,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "sum",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "max",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Max, all pods",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorValue": false,
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"datasource": "Hawk-DS",
"editable": true,
"error": false,
"format": "bytes",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"height": "100px",
"id": 14,
"interval": null,
"isNew": true,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"repeatIteration": 1476706310439,
"scopedVars": {},
"span": 2,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [
{
"queryBy": "tags",
"rate": false,
"refId": "A",
"seriesAggFn": "avg",
"tags": [
{
"name": "container_name",
"value": "$app"
},
{
"name": "descriptor_name",
"value": "memory/usage"
}
],
"target": "select metric",
"tagsQL": "container_name IN [$app] AND descriptor_name='memory/usage'",
"timeAggFn": "max",
"type": "gauge"
}
],
"thresholds": "140000000,180000000",
"title": "Max per pod",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "avg"
}
],
"repeat": "app",
"scopedVars": {
"app": {
"text": "aloha",
"value": "aloha",
"selected": true
}
},
"title": "New row"
}
],
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"templating": {
"list": [
{
"current": {},
"datasource": "Hawk-DS",
"hide": 0,
"includeAll": true,
"label": "Application",
"multi": true,
"name": "app",
"options": [],
"query": "tags/container_name:*",
"refresh": 1,
"regex": "",
"type": "query"
}
]
},
"annotations": {
"list": []
},
"schemaVersion": 12,
"version": 32,
"links": [],
"gnetId": null
}
You should put a YAML file pointing to the JSON files in that folder. For example write /etc/grafana/provisioning/dashboards/local.yml:
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards
options:
path: /var/lib/grafana/dashboards
And then write your JSON file to /var/lib/grafana/dashboards/openshift.json.
Before Grafana 5, my previous solution was to wrap the whole Docker process in a script that uses the API to create the dashboard once the Docker container is up. You can use the GF_SECURITY_ADMIN_PASSWORD environment variable to set the password. You can also use GF_AUTH_ANONYMOUS_ENABLED, but you'll need to make sure it's not accessible to the outside world.
docker run -p 3000:3000 -e GF_AUTH_ANONYMOUS_ENABLED=true grafana/grafana ...
sleep 10 # wait for grafana to load (use a better method of waiting in production)
curl -skfS -XPOST --header "Content-Type: application/json" "http://localhost:3000/grafana/api/dashboards/db" --data-binary #dashboard.json
I've faced with the same problem and I hope the next solution could help somebody.
For example, you have the next working directory:
├── docker-compose.yml
├── grafana
│ ├── Dockerfile
│ ├── dashboards
│ │ └── exported_dashboard.json #dashboard you saved before
│ └── provisioning
│ │
│ ├── dashboards
│ │ └── all.yml
│ └── datasources
│ └── all.yml
In the Dockerfile you have to add provisioning and dashboards folders into grafana container:
FROM grafana/grafana:latest
ADD ./provisioning /etc/grafana/provisioning
ADD ./dashboards /var/lib/grafana/dashboards
In provisioning/dashboards/all.yml you have to set path to imported dashboards folder
where exported_dashboard.json is actually placed:
- name: 'default'
org_id: 1
folder: ''
type: 'file'
options:
folder: '/var/lib/grafana/dashboards'
[Optional] Then you also can configure default datasourse in provisioning/datasources/all.yml, prometheus, for example:
datasources:
- access: 'proxy'
editable: true
is_default: true
name: 'prom1'
org_id: 1
type: 'prometheus'
url: 'http://prometheus:9090'
version: 1
Finally, docker-compose.yml:
version: '3.8'
services:
grafana:
image: awesome-grafana:latest
build:
context: grafana
ports:
- "3000:3000"
environment:
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: "/var/lib/grafana/dashboards/exported_dashboard.json"
As for HOME dashborad, you can use the GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH evironment variable, as shown above.
More about GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH evironment variable you can read in that github issue.
The full comprehensive code example you can find in the next github repository
Original article with explanation there.

How to fetch specific details from the result of twitter api?

With twitter api named GET friends/list, https://api.twitter.com/1.1/friends/list.json i could get details of every user i am following. What i need is some particular data, 'geo' and 'coordinates' to be specific. There do exist a parameter named 'skip_status=true' to skip the status. But please let me know if either 'geo and coordinates' or 'status' alone could be fetched.
"users": [
{
"id": 3233xxxxxx,
"id_str": "3233428303",
"name": "Mars",
"screen_name": "Mars9411",
"location": "",
"description": "",
"url": null,
"entities": {
"description": {
"urls": []
}
},
"protected": false,
"followers_count": 1,
"friends_count": 0,
"listed_count": 0,
"created_at": "Tue Jun 02 09:58:03 +0000 2015",
"favourites_count": 0,
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"verified": false,
"statuses_count": 9,
"lang": "en",
"status": {
"created_at": "Tue Oct 27 12:31:11 +0000 2015",
"id": 6589842841878xxxx,
"id_str": "65898428418781xxxx",
"text": "Hey, Download the app in your phone and you can have an Emergency Contact for FREE. It's Great.\nhttps://t.co/kEZg",
"source": "iOS",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"geo": {
"type": "Point",
"coordinates": [
28.612486,
77.377425
]
},
"coordinates": {
"type": "Point",
"coordinates": [
77.377425,
28.612486
]
},
"place": {
"id": "2e6064382c71b343",
"url": "https://api.twitter.com/1.1/geo/id/2e606438.json",
"place_type": "city",
"name": "Noida",
"full_name": "Noida, Uttar Pradesh",
"country_code": "IN",
"country": "India",
"contained_within": [],
"bounding_box": {
"type": "Polygon",
"coordinates": [
[
[
77.302126,
28.4439811
],
[
77.6953257,
28.4439811
],
[
77.6953257,
28.6471308
],
[
77.302126,
28.6471308
]
]
]
},
"attributes": {}
},
"contributors": null,
"retweet_count": 0,
"favorite_count": 0,
"entities": {
"hashtags": [],
"symbols": [],
"user_mentions": [],
"urls": [
{
"url": "https://t.co/kEZ",
"expanded_url": "http://www.sav.net/apps",
"display_url": "sav.net/apps",
"indices": [
101,
124
]
}
]
},
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"lang": "en"
},
"contributors_enabled": false,
"is_translator": false,
"is_translation_enabled": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_image_url": "http://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png",
"profile_image_url_https": "https://abs.twimg.com/sticky/default_profile_images/default_profile_0_normal.png",
"profile_link_color": "0084B4",
"profile_sidebar_border_color": "C0DEED",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"has_extended_profile": false,
"default_profile": true,
"default_profile_image": true,
"following": true,
"follow_request_sent": false,
"notifications": false,
"muting": false,
"blocking": false,
"blocked_by": false
},
{
"id": 741xxxxxx,
"id_str": "741xxxxxx",
"name": "Food",
"screen_name": "food",
"location": "Fort, IN",
"description": "Locally xxxxx",
"url": "http://t.co",
"entities": {
"url": {
"urls": [
{
"url": "http://t.co",
"expanded_url": "http://Food.com",
"display_url": "food.com",
"indices": [
0,
22
]
}
]
},
"description": {
"urls": []
}
},
"protected": false,
"followers_count": 1976,
"friends_count": 375,
"listed_count": 44,
"created_at": "Tue Aug 07 02:10:46 +0000 2012",
"favourites_count": 247,
"utc_offset": -14400,
"time_zone": "Eastern Time (US & Canada)",
"geo_enabled": true,
"verified": false,
"statuses_count": 1349,
"lang": "en",
"status": {
"created_at": "Sun Oct 25 01:52:10 +0000 2015",
"id": 658098694xxxxxxxxx,
"id_str": "658098694xxxxxxxxx",
"text": "#BenjamenKelly tonight!",
"source": "Twitter for iPhone",
"truncated": false,
"in_reply_to_status_id": 657368006075486200,
"in_reply_to_status_id_str": "657368006075486208",
"in_reply_to_user_id": 509529431,
"in_reply_to_user_id_str": "509529431",
"in_reply_to_screen_name": "Kelly",
"geo": null,
"coordinates": null,
"place": {
"id": "3877d6c867447819",
"url": "https://api.twitter.com/1.1/geo/id/3877d6c867447819.json",
"place_type": "city",
"name": "Fort",
"full_name": "Fort, IN",
"country_code": "US",
"country": "United States",
"contained_within": [],
"bounding_box": {
"type": "Polygon",
"coordinates": [
[
[
-85.336872,
40.95926
],
[
-85.003231,
40.95926
],
[
-85.003231,
41.214574
],
[
-85.336872,
41.214574
]
]
]
},
"attributes": {}
},
"contributors": null,
"retweet_count": 0,
"favorite_count": 1,
"entities": {
"hashtags": [],
"symbols": [],
"user_mentions": [
{
"screen_name": "Benj",
"name": "boy",
"id": 509529431,
"id_str": "509529431",
"indices": [
0,
14
]
}
],
"urls": []
},
"favorited": false,
"retweeted": false,
"lang": "en"
},
"contributors_enabled": false,
"is_translator": false,
"is_translation_enabled": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_image_url": "http://pbs.twimg.com/profile_images/2479745010/yz4eu6wrca8c5e2hw6x9_normal.jpeg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/2479745010/yz4eu6wrca8c5e2hw6x9_normal.jpeg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/741935389/1444260221",
"profile_link_color": "0084B4",
"profile_sidebar_border_color": "C0DEED",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"has_extended_profile": false,
"default_profile": true,
"default_profile_image": false,
"following": true,
"follow_request_sent": false,
"notifications": false,
"muting": false,
"blocking": false,
"blocked_by": false
}
This is what i am getting(supposing i have only 2 users in my following). What i need is as follows:
"geo": {
"type": "Point",
"coordinates": [
28.612486,
77.377425
]
},
"coordinates": {
"type": "Point",
"coordinates": [
77.377425,
28.612486
]
},
"geo": null,
"coordinates": null,
You need to filter out those details from the results yourself. The API will return the complete person object, not specific fields.

Resources