Related
I am new bee to python and trying to use 'confluent_kafka' for avro message produce.
Using 'confluent_kafka.schema_registry.avro.AvroSerializer' for the same
(referred : https://github.com/confluentinc/confluent-kafka-python/blob/master/examples/avro_producer.py)
It works for simple avro schema with dict(json converted to dict) input, but for below sample schema I am getting error :
Schema :
{
"type": "record",
"name": "Envelope",
"namespace": "CoreOLTPEvents.dbo.Event",
"fields": [{
"name": "before",
"type": ["null", {
"type": "record",
"name": "Value",
"fields": [{
"name": "EventId",
"type": "long"
}, {
"name": "CameraId",
"type": ["null", "long"],
"default": null
}],
"connect.name": "CoreOLTPEvents.dbo.Event.Value"
}],
"default": null
}, {
"name": "after",
"type": ["null", "Value"],
"default": null
}, {
"name": "source",
"type": {
"type": "record",
"name": "Source",
"namespace": "io.debezium.connector.sqlserver",
"fields": [{
"name": "version",
"type": "string"
}, {
"name": "connector",
"type": "string"
}],
"connect.name": "io.debezium.connector.sqlserver.Source"
}
}, {
"name": "op",
"type": "string"
}],
"connect.name": "CoreOLTPEvents.dbo.Event.Envelope"
}
Input Json :
{
"after": null,
"before": {
"CoreOLTPEvents.dbo.Event.Value" : {
"EventId": 1111111111,
"CameraId": 222222222
}
},
"source": {
"version": "InitialLoad",
"connector": "sqlserver"
},
"op": "C"
}
Error :
ValueError: {'CoreOLTPEvents.dbo.Event.Value': {'EventId': 1111111111, 'CameraId': 222222222}} (type <class 'dict'>) do not match ['null', {'connect.name': 'CoreOLTPEvents.dbo.Event.Value', 'type': 'record', 'name': 'CoreOLTPEvents.dbo.Event.Value', 'fields': [{'name': 'EventId', 'type': 'long'}, {'default': None, 'name': 'CameraId', 'type': ['null', 'long']}]}] on field before
'before' field type is union (['null',record]), if I change it to only record (remove union) then it works fine.
But I need to adjust my input such a way that it works for given schema.
(Note : I am reading json input using 'json.load(json_file)' so it gives dict output)
Any help would be much appreciated.
Update :
Actual large schema :
{
"type": "record",
"name": "Envelope",
"namespace": "CoreOLTPEvents.dbo.Event",
"fields": [{
"name": "before",
"type": ["null", {
"type": "record",
"name": "Value",
"fields": [{
"name": "EventId",
"type": "long"
}, {
"name": "CameraId",
"type": ["null", "long"],
"default": null
}, {
"name": "SiteId",
"type": ["null", "long"],
"default": null
}, {
"name": "VehicleId",
"type": ["null", "long"],
"default": null
}, {
"name": "EventReviewStatusID",
"type": "int"
}, {
"name": "EventTypeId",
"type": ["null", "int"],
"default": null
}, {
"name": "EventDateTime",
"type": ["null", {
"type": "string",
"connect.name": "net.smartdrive.converters.SmartdriveEventDateFieldConverter"
}],
"default": null
}, {
"name": "FTPUploadDateTime",
"type": {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}
}, {
"name": "CAMFileName",
"type": "string"
}, {
"name": "KeypadEntryCode",
"type": ["null", "string"],
"default": null
}, {
"name": "IsActive",
"type": {
"type": "boolean",
"connect.default": true
},
"default": true
}, {
"name": "Flagged",
"type": "boolean"
}, {
"name": "EventTitle",
"type": ["null", "string"],
"default": null
}, {
"name": "CreatedBy",
"type": "long"
}, {
"name": "CreatedDate",
"type": {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}
}, {
"name": "ModifiedBy",
"type": "long"
}, {
"name": "ModifiedDate",
"type": {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}
}, {
"name": "ReReviewAnalysis",
"type": ["null", "string"],
"default": null
}, {
"name": "LegacyEventId",
"type": ["null", "long"],
"default": null
}, {
"name": "TripId",
"type": ["null", "long"],
"default": null
}, {
"name": "FileVersion",
"type": ["null", "string"],
"default": null
}, {
"name": "EventNumber",
"type": ["null", "string"],
"default": null
}, {
"name": "Latitude",
"type": ["null", {
"type": "bytes",
"scale": 10,
"precision": 13,
"connect.version": 1,
"connect.parameters": {
"scale": "10",
"connect.decimal.precision": "13"
},
"connect.name": "org.apache.kafka.connect.data.Decimal",
"logicalType": "decimal"
}],
"default": null
}, {
"name": "Longitude",
"type": ["null", {
"type": "bytes",
"scale": 10,
"precision": 13,
"connect.version": 1,
"connect.parameters": {
"scale": "10",
"connect.decimal.precision": "13"
},
"connect.name": "org.apache.kafka.connect.data.Decimal",
"logicalType": "decimal"
}],
"default": null
}, {
"name": "GeoAddressId",
"type": ["null", "long"],
"default": null
}, {
"name": "ReviewedEventId",
"type": ["null", "long"],
"default": null
}, {
"name": "VideoStatus",
"type": {
"type": "int",
"connect.default": 0
},
"default": 0
}, {
"name": "PredictionImportance",
"type": ["null", {
"type": "bytes",
"scale": 10,
"precision": 15,
"connect.version": 1,
"connect.parameters": {
"scale": "10",
"connect.decimal.precision": "15"
},
"connect.name": "org.apache.kafka.connect.data.Decimal",
"logicalType": "decimal"
}],
"default": null
}, {
"name": "FlaggedBy",
"type": ["null", "long"],
"default": null
}, {
"name": "FlaggedDate",
"type": ["null", {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}],
"default": null
}, {
"name": "TriggerTypeId",
"type": ["null", "int"],
"default": null
}, {
"name": "VideoDeleteDate",
"type": ["null", {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}],
"default": null
}, {
"name": "MetadataDeleteDate",
"type": ["null", {
"type": "long",
"connect.version": 1,
"connect.name": "io.debezium.time.Timestamp"
}],
"default": null
}, {
"name": "RetentionStatus",
"type": {
"type": "int",
"connect.default": 0,
"connect.type": "int16"
},
"default": 0
}, {
"name": "PartnerTriggerId",
"type": ["null", "int"],
"default": null
}, {
"name": "CoachingStateId",
"type": {
"type": "int",
"connect.default": 0,
"connect.type": "int16"
},
"default": 0
}, {
"name": "EventKudoHistoryId",
"type": ["null", "int"],
"default": null
}],
"connect.name": "CoreOLTPEvents.dbo.Event.Value"
}],
"default": null
}, {
"name": "after",
"type": ["null", "Value"],
"default": null
}, {
"name": "source",
"type": {
"type": "record",
"name": "Source",
"namespace": "io.debezium.connector.sqlserver",
"fields": [{
"name": "version",
"type": "string"
}, {
"name": "connector",
"type": "string"
}, {
"name": "name",
"type": "string"
}, {
"name": "ts_ms",
"type": "long"
}, {
"name": "snapshot",
"type": [{
"type": "string",
"connect.version": 1,
"connect.parameters": {
"allowed": "true,last,false"
},
"connect.default": "false",
"connect.name": "io.debezium.data.Enum"
}, "null"],
"default": "false"
}, {
"name": "db",
"type": "string"
}, {
"name": "schema",
"type": "string"
}, {
"name": "table",
"type": "string"
}, {
"name": "change_lsn",
"type": ["null", "string"],
"default": null
}, {
"name": "commit_lsn",
"type": ["null", "string"],
"default": null
}, {
"name": "event_serial_no",
"type": ["null", "long"],
"default": null
}],
"connect.name": "io.debezium.connector.sqlserver.Source"
}
}, {
"name": "op",
"type": "string"
}, {
"name": "ts_ms",
"type": ["null", "long"],
"default": null
}, {
"name": "transaction",
"type": ["null", {
"type": "record",
"name": "ConnectDefault",
"namespace": "io.confluent.connect.avro",
"fields": [{
"name": "id",
"type": "string"
}, {
"name": "total_order",
"type": "long"
}, {
"name": "data_collection_order",
"type": "long"
}]
}],
"default": null
}],
"connect.name": "CoreOLTPEvents.dbo.Event.Envelope"
}
Input for large schema :
{
"before": null,
"after": {
"EventId": 1234566,
"CameraId": 2233,
"SiteId": 111,
"VehicleId": 45587,
"EventReviewStatusID": 10,
"EventTypeId": 123,
"EventDateTime": "2015-01-02T01:30:29Z",
"FTPUploadDateTime": 1420193330590,
"CAMFileName": "XYZ",
"KeypadEntryCode": "0",
"IsActive": false,
"Flagged": false,
"EventTitle": null,
"CreatedBy": 1,
"CreatedDate": 1420191120730,
"ModifiedBy": 1,
"ModifiedDate": 1577871185680,
"ReReviewAnalysis": null,
"LegacyEventId": null,
"TripId": 3382,
"FileVersion": "2.2",
"EventNumber": "AAAA-BBBB",
"Latitude": "UU9elrA=",
"Longitude": "/ueZUeFw",
"GeoAddressId": null,
"ReviewedEventId": 129411077,
"VideoStatus": 4,
"PredictionImportance": 0.1402457539,
"FlaggedBy": null,
"FlaggedDate": null,
"TriggerTypeId": 322,
"VideoDeleteDate": 1422783120000,
"MetadataDeleteDate": 1577871120000,
"RetentionStatus": 15,
"PartnerTriggerId": null,
"CoachingStateId": 0,
"EventKudoHistoryId": null
},
"source": {
"version": "Final",
"connector": "sqlserver",
"name": "CoreOLTP",
"ts_ms": 1615813992548,
"snapshot": "false",
"db": "CoreOLTP",
"schema": "dbo",
"table": "xyz",
"change_lsn": null,
"commit_lsn": null,
"event_serial_no": null
},
"op": "C",
"ts_ms": 1615813992548,
"transaction": null
}
Error :
confluent_kafka.error.ValueSerializationError: KafkaError{code=_VALUE_SERIALIZATION,val=-161,str="{'EventId': 129411077, 'CameraId': 46237, 'SiteId': 2148, 'VehicleId': 45587, 'EventReviewStatusID': 10, 'EventTypeId': 247, 'EventDateTime': '2015-01-02T01:30:29Z', 'FTPUploadDateTime': 1420191120590, 'CAMFileName': 'JD2BC02120150102013029ER.SDE', 'KeypadEntryCode': '0', 'IsActive': False, 'Flagged': False, 'EventTitle': None, 'CreatedBy': 1, 'CreatedDate': 1420191120730, 'ModifiedBy': 1, 'ModifiedDate': 1577871185680, 'ReReviewAnalysis': None, 'LegacyEventId': None, 'TripId': 3382, 'FileVersion': '2.2', 'EventNumber': 'WSHX-8QQ2', 'Latitude': 'UU9elrA=', 'Longitude': '/ueZUeFw', 'GeoAddressId': None, 'ReviewedEventId': 129411077, 'VideoStatus': 4, 'PredictionImportance': 0.1402457539, 'FlaggedBy': None, 'FlaggedDate': None, 'TriggerTypeId': 322, 'VideoDeleteDate': 1422783120000, 'MetadataDeleteDate': 1577871120000, 'RetentionStatus': 15, 'PartnerTriggerId': None, 'CoachingStateId': 0, 'EventKudoHistoryId': None} (type <class 'dict'>) do not match ['null', 'CoreOLTPEvents.dbo.Event.Value'] on field after"}
You just need to change your input so that the before field doesn't have the namespace. So it needs to look like this:
{
"after": null,
"before": {
"EventId": 1111111111,
"CameraId": 222222222
},
"source": {
"version": "InitialLoad",
"connector": "sqlserver"
},
"op": "C"
}
The original input you had looked like it was trying to be JSON encoded avro because the field before had the CoreOLTPEvents.dbo.Event.Value namespace. However, I'm guessing it must have been hand crafted because CameraId should have been specified as {"long": 222222222} rather than just 222222222.
If you do actually have Avro encoded JSON (from the result of some other process or something) then you you could use something like fastavro.json_reader to read in that file and it will create the correct memory representation (that doesn't include the type information for union fields).
UPDATE:
To figure out what the problem is with the full schema and full data, I first loaded the two objects using json.load and then used fastavro.validate(record, schema) The output from that is a stacktrace that ends with this:
fastavro._validate_common.ValidationError: [
"CoreOLTPEvents.dbo.Event.Envelope.after is <{'EventId': 1234566, 'CameraId': 2233, 'SiteId': 111, 'VehicleId': 45587, 'EventReviewStatusID': 10, 'EventTypeId': 123, 'EventDateTime': '2015-01-02T01:30:29Z', 'FTPUploadDateTime': 1420193330590, 'CAMFileName': 'XYZ', 'KeypadEntryCode': '0', 'IsActive': False, 'Flagged': False, 'EventTitle': None, 'CreatedBy': 1, 'CreatedDate': 1420191120730, 'ModifiedBy': 1, 'ModifiedDate': 1577871185680, 'ReReviewAnalysis': None, 'LegacyEventId': None, 'TripId': 3382, 'FileVersion': '2.2', 'EventNumber': 'AAAA-BBBB', 'Latitude': 'UU9elrA=', 'Longitude': '/ueZUeFw', 'GeoAddressId': None, 'ReviewedEventId': 129411077, 'VideoStatus': 4, 'PredictionImportance': 0.1402457539, 'FlaggedBy': None, 'FlaggedDate': None, 'TriggerTypeId': 322, 'VideoDeleteDate': 1422783120000, 'MetadataDeleteDate': 1577871120000, 'RetentionStatus': 15, 'PartnerTriggerId': None, 'CoachingStateId': 0, 'EventKudoHistoryId': None}> of type <class 'dict'> expected null",
"CoreOLTPEvents.dbo.Event.Value.Latitude is <UU9elrA=> of type <class 'str'> expected null",
"CoreOLTPEvents.dbo.Event.Value.Latitude is <UU9elrA=> of type <class 'str'> expected {'scale': 10, 'precision': 13, 'connect.version': 1, 'connect.parameters': {'scale': '10', 'connect.decimal.precision': '13'}, 'connect.name': 'org.apache.kafka.connect.data.Decimal', 'logicalType': 'decimal', 'type': 'bytes'}"
]
So that is trying to tell us that there is 3 potential problems. The first is that the value in after doesn't match null, but we can ignore that because we don't want after to match null.
The later two problems are the actual problem. It says that the value of Latitude is the string UU9elrA=, but that doesn't match either null or bytes. The string here looks base64 encoded, so maybe you have some code that decodes that to bytes and if so then maybe the actual problem is something else, but if so then I think you should be able to use fastavro.validate to figure out what the problem is.
The following has been tried using the Microsoft Graph API Graph Explorer to eliminate any third-party causes. There is an existing event in an Outlook calendar:
{
"#odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('WWWW')/calendars('XXXX')/events/$entity",
"#odata.etag": "W/\"wIqdPsepnEOtI+GbtqRT1gAAElvUsQ==\"",
"id": "YYYY",
"createdDateTime": "2020-06-18T05:19:22.5766648Z",
"lastModifiedDateTime": "2020-06-18T17:17:23.3143899Z",
"changeKey": "wIqdPsepnEOtI+GbtqRT1gAAElvUsQ==",
"categories": [],
"originalStartTimeZone": "UTC",
"originalEndTimeZone": "UTC",
"iCalUId": "ZZZZ",
"reminderMinutesBeforeStart": 15,
"isReminderOn": false,
"hasAttachments": false,
"subject": "Review proposal",
"bodyPreview": "",
"importance": "normal",
"sensitivity": "normal",
"isAllDay": false,
"isCancelled": false,
"isOrganizer": true,
"responseRequested": true,
"seriesMasterId": null,
"showAs": "busy",
"type": "singleInstance",
"webLink": "https://outlook.office365.com/owa/?itemid=YYY&exvsurl=1&path=/calendar/item",
"onlineMeetingUrl": null,
"isOnlineMeeting": false,
"onlineMeetingProvider": "unknown",
"allowNewTimeProposals": true,
"recurrence": null,
"onlineMeeting": null,
"responseStatus": {
"response": "organizer",
"time": "0001-01-01T00:00:00Z"
},
"body": {
"contentType": "text",
"content": ""
},
"start": {
"dateTime": "2020-06-26T14:30:00.0000000",
"timeZone": "UTC"
},
"end": {
"dateTime": "2020-06-26T15:00:00.0000000",
"timeZone": "UTC"
},
"location": {
"displayName": "",
"locationType": "default",
"uniqueIdType": "unknown",
"address": {},
"coordinates": {}
},
"locations": [],
"attendees": [
{
"type": "required",
"status": {
"response": "none",
"time": "0001-01-01T00:00:00Z"
},
"emailAddress": {
"name": "attendee#example.com",
"address": "attendee#example.com"
}
},
{
"type": "required",
"status": {
"response": "none",
"time": "0001-01-01T00:00:00Z"
},
"emailAddress": {
"name": "Bob Smith",
"address": "organizer#example.com"
}
}
],
"organizer": {
"emailAddress": {
"name": "Bob Smith",
"address": "organizer#example.com"
}
}
}
The attendee's name was not known at the time the event was created. attendee#example.com was given which produced an attendee whose name and email address were both attendee#example.com. Later, the attendee's name became known. A patch request is sent to the organizer's calendar through Microsoft Graph API to update the event with the goal of overwriting the attendee's name, but leaving the email address as is.
{
"#odata.type": "#microsoft.graph.event",
"attendees": [{
"status": {
"response": "none",
"time": "0001-01-01T00:00:00Z"
},
"type": "required",
"emailAddress": {
"address": "attendee#example.com",
"name": "Julie Williams"
}
}, {
"status": {
"response": "none"
},
"type": "required",
"emailAddress": {
"address": "organizer#example.com",
"name": "Bob Smith"
}
}],
"createdDateTime": "2020-06-18T05:19:22.5766648Z",
"end": {
"dateTime": "2020-06-26T15:00:00",
"timeZone": "UTC"
},
"iCalUId": "ZZZZ",
"id": "YYYY",
"importance": "normal",
"isOrganizer": true,
"lastModifiedDateTime": "2020-06-18T05:19:23.7881324Z",
"location": {},
"organizer": {
"emailAddress": {
"address": "organizer#example.com",
"name": "Bob Smith"
}
},
"originalEndTimeZone": "Eastern Standard Time",
"originalStartTimeZone": "Eastern Standard Time",
"reminderMinutesBeforeStart": 15,
"responseRequested": true,
"responseStatus": {
"response": "organizer",
"time": "0001-01-01T00:00:00Z"
},
"sensitivity": "normal",
"showAs": "busy",
"start": {
"dateTime": "2020-06-26T14:30:00",
"timeZone": "UTC"
},
"subject": "Review proposal",
"type": "singleInstance",
"webLink": "https://outlook.office365.com/owa/?itemid=YYYY&exvsurl=1&path=/calendar/item"
}
The response shows that the attendee's name did not change.
{
"#odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('WWWW')/calendars('XXXX')/events/$entity",
"#odata.etag": "W/\"wIqdPsepnEOtI+GbtqRT1gAAElvUsQ==\"",
"id": "YYYY",
"createdDateTime": "2020-06-18T05:19:22.5766648Z",
"lastModifiedDateTime": "2020-06-18T17:17:23.3143899Z",
"changeKey": "wIqdPsepnEOtI+GbtqRT1gAAElvUsQ==",
"categories": [],
"originalStartTimeZone": "UTC",
"originalEndTimeZone": "UTC",
"iCalUId": "ZZZZ",
"reminderMinutesBeforeStart": 15,
"isReminderOn": false,
"hasAttachments": false,
"subject": "Review proposal",
"bodyPreview": "",
"importance": "normal",
"sensitivity": "normal",
"isAllDay": false,
"isCancelled": false,
"isOrganizer": true,
"responseRequested": true,
"seriesMasterId": null,
"showAs": "busy",
"type": "singleInstance",
"webLink": "https://outlook.office365.com/owa/?itemid=YYYY&exvsurl=1&path=/calendar/item",
"onlineMeetingUrl": null,
"isOnlineMeeting": false,
"onlineMeetingProvider": "unknown",
"allowNewTimeProposals": true,
"recurrence": null,
"onlineMeeting": null,
"responseStatus": {
"response": "organizer",
"time": "0001-01-01T00:00:00Z"
},
"body": {
"contentType": "text",
"content": ""
},
"start": {
"dateTime": "2020-06-26T14:30:00.0000000",
"timeZone": "UTC"
},
"end": {
"dateTime": "2020-06-26T15:00:00.0000000",
"timeZone": "UTC"
},
"location": {
"displayName": "",
"locationType": "default",
"uniqueIdType": "unknown",
"address": {},
"coordinates": {}
},
"locations": [],
"attendees": [
{
"type": "required",
"status": {
"response": "none",
"time": "0001-01-01T00:00:00Z"
},
"emailAddress": {
"name": "attendee#example.com",
"address": "attendee#example.com"
}
},
{
"type": "required",
"status": {
"response": "none",
"time": "0001-01-01T00:00:00Z"
},
"emailAddress": {
"name": "Bob Smith",
"address": "organizer#example.com"
}
}
],
"organizer": {
"emailAddress": {
"name": "Bob Smith",
"address": "organizer#example.com"
}
}
}
Should I expect the attendee's name to be updated? Is there another way to do it?
Is it possible to get the List of all pull request with the details like the date when it was raised, the Person who raised it and the list of all the approvers from the BITBUCKET SERVER using the REST APIs
Absolutely, You can fetch the list of Pull requests along with all the other details like who raised it, when it was raised and who are the reviewers using the following API,
https://docs.atlassian.com/bitbucket-server/rest/7.4.0/bitbucket-rest.html#idp285
You will have the following as response,
{
"size": 1,
"limit": 25,
"isLastPage": true,
"values": [
{
"id": 101,
"version": 1,
"title": "Talking Nerdy",
"description": "It’s a kludge, but put the tuple from the database in the cache.",
"state": "OPEN",
"open": true,
"closed": false,
"createdDate": 1359075920,
"updatedDate": 1359085920,
"fromRef": {
"id": "refs/heads/feature-ABC-123",
"repository": {
"slug": "my-repo",
"name": null,
"project": {
"key": "PRJ"
}
}
},
"toRef": {
"id": "refs/heads/master",
"repository": {
"slug": "my-repo",
"name": null,
"project": {
"key": "PRJ"
}
}
},
"locked": false,
"author": {
"user": {
"name": "tom",
"emailAddress": "tom#example.com",
"id": 115026,
"displayName": "Tom",
"active": true,
"slug": "tom",
"type": "NORMAL"
},
"role": "AUTHOR",
"approved": true,
"status": "APPROVED"
},
"reviewers": [
{
"user": {
"name": "jcitizen",
"emailAddress": "jane#example.com",
"id": 101,
"displayName": "Jane Citizen",
"active": true,
"slug": "jcitizen",
"type": "NORMAL"
},
"lastReviewedCommit": "7549846524f8aed2bd1c0249993ae1bf9d3c9998",
"role": "REVIEWER",
"approved": true,
"status": "APPROVED"
}
],
"participants": [
{
"user": {
"name": "dick",
"emailAddress": "dick#example.com",
"id": 3083181,
"displayName": "Dick",
"active": true,
"slug": "dick",
"type": "NORMAL"
},
"role": "PARTICIPANT",
"approved": false,
"status": "UNAPPROVED"
},
{
"user": {
"name": "harry",
"emailAddress": "harry#example.com",
"id": 99049120,
"displayName": "Harry",
"active": true,
"slug": "harry",
"type": "NORMAL"
},
"role": "PARTICIPANT",
"approved": true,
"status": "APPROVED"
}
],
"links": {
"self": [
{
"href": "http://link/to/pullrequest"
}
]
}
}
],
"start": 0 }
I am creating an avro schema for a JSON payload that appear to have an array of multiple objects. I'm not sure exactly how to represent this in the schema. The key in question is content:
{
"id": "channel-id",
"name": "My Channel with a New Title",
"description": "Herpy me derpy merpus herpsum ner berp berps derp ter tee",
"privacyLevel": "<private|org>",
"planId": "some-plan-id",
"owner": "a-user-handle",
"curators": [
"user-handle-1",
"user-handle-2"
],
"members": 5,
"content": [
{
"id": "docker",
"slug": "docker",
"index": 1,
"type": "path"
},
{
"id": "such-linkage",
"slug": "such-linkage",
"index": 2,
"type": "external-link",
"details": {
"url": "http://some-dank-link.com",
"title": "My Dank Link",
"contentType": "External Link",
"level": "Beginner",
"duration": "PT34293H33M9S"
}
},
{
"id": "21f1e812-b10a-40df-8b52-3a1d05fc215c",
"slug": "windows-azure-storage-in-depth",
"index": 3,
"type": "course"
},
{
"id": "7c346c05-6416-42dd-80b2-d5e758de7926",
"slug": "7c346c05-6416-42dd-80b2-d5e758de7926",
"index": 4,
"type": "project"
}
],
"imageUrls": ["https://url/to/an/image", "https://url/to/another/image"],
"analyticsEnabled": true,
"orgDiscoverable": false,
"createdDate": "2015-12-31T01:23:45+00:00",
"archiveDate": "2015-12-31T01:23:45+00:00",
"messagePublishedAt": "2015-12-31T01:23:45+00:00"
}
If you are asking if it is possible create an array with different kind of records, it is. Avro support this through union. it would looks like .
{
"name": "myRecord",
"type":"record",
"fields":[
{
"name":"myArrayWithMultiplesTypes",
"type":{
"type": "array",
"items":[
{
"name":"typeOne",
"type":"record",
"fields":[
{"name":"name", "type":"string"}
]
},
{
"name":"typeTwo",
"type":"record",
"fields":[
{"name":"id", "type":"int"}
]
}
]
}
}
]
}
If you already have the records defined previously, then it could look like this:
{
"name": "mulitplePossibleTypes",
"type": [
"null",
{
"type": "array",
"items": [
"com.xyz.kola.cloud.events.itemmanager.Part",
"com.xyz.kola.cloud.events.itemmanager.Document",
"com.xyz.kola.cloud.events.itemmanager.DigitalModel",
"com.xyz.kola.cloud.events.itemmanager.Interface"
]
}
]
},
Using Neo4J 2.0.0-M5, I'm trying to create my database using a batch, as is explained at :
http://docs.neo4j.org/chunked/milestone/rest-api-batch-ops.html
What I want to do is inserting nodes and relationships with the "get_or_create" mode, so that I can run the batch multiple times without creating redundant nodes and relationships.
Creating unique nodes can be done with /index/node/indexname?uniqueness=get_or_create, for example :
POST /db/data/batch [
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":0,"body":{"key":"nom", "value":"organisation", "properties": {"nom":"organisation"}}},
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":1,"body":{"key":"nom", "value":"établissement", "properties": {"nom":"établissement"}}}
]
However, I don't find how I can create a unique relationship between the two indexed nodes?
I want to state that there is a "est" relationship between "établissement" and "organisation". If I try with :
POST /db/data/batch [
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":0,"body":{"key":"nom", "value":"organisation", "properties": {"nom":"organisation"}}},
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":1,"body":{"key":"nom", "value":"établissement", "properties": {"nom":"établissement"}}},
{"method":"POST","to":"{1}/relationships","body":{"to":"{0}","type":"est"}},
{"method":"POST","to":"/index/relationship/my_rels?uniqueness=get_or_create","body":{"key":"nom","value":"est","uri":"{1}"}}
]
I get :
==> 500 Internal Server Error
==> {
==> "message" : "",
==> "exception" : "BatchOperationFailedException",
==> "fullname" : "org.neo4j.server.rest.domain.BatchOperationFailedException",
==> "stacktrace" : ["org.neo4j.server.rest.batch.NonStreamingBatchOperations.invoke(NonStreamingBatchOperations.java:63)", "org.neo4j.server.rest.batch.BatchOperations.performRequest(BatchOperations.java:188)", "org.neo4j.server.rest.batch.BatchOperations.parseAndPerform(BatchOperations.java:159)", "org.neo4j.server.rest.batch.NonStreamingBatchOperations.performBatchJobs(NonStreamingBatchOperations.java:48)", "org.neo4j.server.rest.web.BatchOperationService.batchProcess(BatchOperationService.java:123)", "org.neo4j.server.rest.web.BatchOperationService.performBatchOperations(BatchOperationService.java:73)", "java.lang.reflect.Method.invoke(Method.java:606)", "org.neo4j.server.rest.transactional.TransactionalRequestDispatcher.dispatch(TransactionalRequestDispatcher.java:132)", "org.neo4j.server.rest.security.SecurityFilter.doFilter(SecurityFilter.java:112)" ]
==> }
I'm stuck with this problem, so any advice would be really appreciated. Thank you!
Grégoire
EDIT: the problem seems to come from a bug that prevents unique nodes to be referenced from within a batch :
https://github.com/neo4j/neo4j/issues/84
The workaround (as stated in the github bug you mentioned) is to use Cypher.
Example using Batch and Cypher:
[
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":0,"body":{"key":"nom", "value":"organisation", "properties": {"nom":"organisation"}}},
{"method":"POST","to":"/index/node/concept?uniqueness=get_or_create","id":1,"body":{"key":"nom", "value":"établissement", "properties": {"nom":"établissement"}}},
{
"method": "POST",
"to": "/cypher",
"id": 1,
"body": {
"query" : "START a=node:concept(nom={aVal}), b=node:concept(nom={bVal}) CREATE b-[r:est]->a RETURN a, b, r",
"params" : {
"aVal" : "établissement",
"bVal" : "organisation"
}
}
}
]
Output:
[{
"id": 0,
"location": "http://localhost:7474/db/data/index/node/concept/nom/organisation/18688",
"body": {
"extensions": {},
"paged_traverse": "http://localhost:7474/db/data/node/18688/paged/traverse/{returnType}{?pageSize,leaseTime}",
"outgoing_relationships": "http://localhost:7474/db/data/node/18688/relationships/out",
"all_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/all/{-list|&|types}",
"traverse": "http://localhost:7474/db/data/node/18688/traverse/{returnType}",
"property": "http://localhost:7474/db/data/node/18688/properties/{key}",
"all_relationships": "http://localhost:7474/db/data/node/18688/relationships/all",
"self": "http://localhost:7474/db/data/node/18688",
"outgoing_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/out/{-list|&|types}",
"properties": "http://localhost:7474/db/data/node/18688/properties",
"incoming_relationships": "http://localhost:7474/db/data/node/18688/relationships/in",
"incoming_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/in/{-list|&|types}",
"create_relationship": "http://localhost:7474/db/data/node/18688/relationships",
"data": {
"nom": "organisation"
},
"indexed": "http://localhost:7474/db/data/index/node/concept/nom/organisation/18688"
},
"from": "/index/node/concept?uniqueness=get_or_create"
}, {
"id": 1,
"location": "http://localhost:7474/db/data/index/node/concept/nom/%EF%BF%BDtablissement/18689",
"body": {
"extensions": {},
"paged_traverse": "http://localhost:7474/db/data/node/18689/paged/traverse/{returnType}{?pageSize,leaseTime}",
"outgoing_relationships": "http://localhost:7474/db/data/node/18689/relationships/out",
"all_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/all/{-list|&|types}",
"traverse": "http://localhost:7474/db/data/node/18689/traverse/{returnType}",
"property": "http://localhost:7474/db/data/node/18689/properties/{key}",
"all_relationships": "http://localhost:7474/db/data/node/18689/relationships/all",
"self": "http://localhost:7474/db/data/node/18689",
"outgoing_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/out/{-list|&|types}",
"properties": "http://localhost:7474/db/data/node/18689/properties",
"incoming_relationships": "http://localhost:7474/db/data/node/18689/relationships/in",
"incoming_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/in/{-list|&|types}",
"create_relationship": "http://localhost:7474/db/data/node/18689/relationships",
"data": {
"nom": "�tablissement"
},
"indexed": "http://localhost:7474/db/data/index/node/concept/nom/%EF%BF%BDtablissement/18689"
},
"from": "/index/node/concept?uniqueness=get_or_create"
}, {
"id": 1,
"body": {
"columns": ["a", "b", "r"],
"data": [
[{
"paged_traverse": "http://localhost:7474/db/data/node/18689/paged/traverse/{returnType}{?pageSize,leaseTime}",
"outgoing_relationships": "http://localhost:7474/db/data/node/18689/relationships/out",
"data": {
"nom": "�tablissement"
},
"traverse": "http://localhost:7474/db/data/node/18689/traverse/{returnType}",
"all_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/all/{-list|&|types}",
"self": "http://localhost:7474/db/data/node/18689",
"all_relationships": "http://localhost:7474/db/data/node/18689/relationships/all",
"property": "http://localhost:7474/db/data/node/18689/properties/{key}",
"properties": "http://localhost:7474/db/data/node/18689/properties",
"outgoing_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/out/{-list|&|types}",
"incoming_relationships": "http://localhost:7474/db/data/node/18689/relationships/in",
"incoming_typed_relationships": "http://localhost:7474/db/data/node/18689/relationships/in/{-list|&|types}",
"extensions": {},
"create_relationship": "http://localhost:7474/db/data/node/18689/relationships"
}, {
"paged_traverse": "http://localhost:7474/db/data/node/18688/paged/traverse/{returnType}{?pageSize,leaseTime}",
"outgoing_relationships": "http://localhost:7474/db/data/node/18688/relationships/out",
"data": {
"nom": "organisation"
},
"traverse": "http://localhost:7474/db/data/node/18688/traverse/{returnType}",
"all_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/all/{-list|&|types}",
"self": "http://localhost:7474/db/data/node/18688",
"all_relationships": "http://localhost:7474/db/data/node/18688/relationships/all",
"property": "http://localhost:7474/db/data/node/18688/properties/{key}",
"properties": "http://localhost:7474/db/data/node/18688/properties",
"outgoing_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/out/{-list|&|types}",
"incoming_relationships": "http://localhost:7474/db/data/node/18688/relationships/in",
"incoming_typed_relationships": "http://localhost:7474/db/data/node/18688/relationships/in/{-list|&|types}",
"extensions": {},
"create_relationship": "http://localhost:7474/db/data/node/18688/relationships"
}, {
"start": "http://localhost:7474/db/data/node/18688",
"data": {},
"self": "http://localhost:7474/db/data/relationship/44187",
"property": "http://localhost:7474/db/data/relationship/44187/properties/{key}",
"properties": "http://localhost:7474/db/data/relationship/44187/properties",
"type": "est",
"extensions": {},
"end": "http://localhost:7474/db/data/node/18689"
}]
]
},
"from": "/cypher"
}]