Openlayers bbox strategy - openlayers-3

I have bbox strategy for one source of data. Code looks like this:
bbox: function newBboxFeatureSource(url, typename) {
return new ol.source.Vector({
loader: function (extent) {
let u = `${url}&TYPENAME=${typename}&bbox=${extent.join(",")}`;
$.ajax(u).then((response) => {
this.addFeatures(
geoJsonFormat.readFeatures(response)
);
});
},
strategy: ol.loadingstrategy.bbox
});
},
I works fine but... When I pan/move the map then this loader is calling again and add another features which fit to new box. But there is a lot of duplicates then because some of new features are just the same as old.
So I wanted first clear all features using this.clear() before add new features but when I add this command then loader is running all the time and I have "infinitive loop". Do you know why? How can I disable loading new features after calling this.clear()?
edit:
my response with features looks like this:
{ "type": "FeatureCollection", "crs": { "type": "name", "properties":
{ "name": "urn:ogc:def:crs:EPSG::3857" } },
"features": [ { "type": "Feature", "properties": { "ogc_fid": "2",
"name": "AL" }, "geometry": { "type": "MultiPolygon" , "coordinates":
[ [ [ ... ] ] ] } }, { "type": "Feature", "properties": { "ogc_fid":
"3", "name": "B" }, "geometry": { "type": "MultiPolygon" ,
"coordinates": [ [ [ ...] ] ] } } ..... and so on
I've removed coordinates because there was too many of them.
My features are generated by mapserver and are configured in .map file which looks like this:
LAYER
NAME "postcode_area_boundaries"
METADATA
"wfs_title" "Postcode area boundaries"
"wfs_srs" "EPSG:3857"
"wfs_enable_request" "*"
"wfs_getfeature_formatlist" "json"
"wfs_geomtype" "multipolygon"
"wfs_typename" "postcode_area_boundaries"
"wms_context_fid" "id"
"wfs_featureid" "id"
"gml_featureid" "id"
"gml_include_items" "id,postarea,wkb_geometry"
"gml_postarea_alias" "name"
"ows_featureid" "id"
"tinyows_table" "postcode_area_boundaries"
"tinyows_retrievable" "1"
"tinyows_include_items" "id,postarea,wkb_geometry"
END
TYPE POLYGON
STATUS ON
CONNECTIONTYPE POSTGIS
CONNECTION "..."
DATA "wkb_geometry FROM postcode_area_boundaries USING UNIQUE id"
DUMP TRUE
END

To summarize the discussion and answer the initial question:
The features sent by the server need an attribute called id, which must be unique and the same for the feature on every request.
{type: "Feature", id: "some-wfs.1234", properties: { "ogc_fid": 2, ...
See this GitHub Issue for the original comment of ahocevar.
In GeoServer this can be achieved if you set an identifier in your layer.
I guess there is something similar to set in MapServer.

Following up on #bennos comment: you can expose the fid in Mapserver using the FORMATOPTION USE_FEATUREID=true as described in: https://mapserver.org/output/ogr_output.html
USE_FEATUREID=true/false
Starting from MapServer v7.0.2. Defaults to false. Include feature ids in the generated output, if the ows_featureid metadata key is set at the layer level. The featureid column to use should be an integer column. Useful if you need to include an “id” attribute to your geojson output. Use with caution as some OGR output drivers may behave strangely when fed with random FIDs.
OUTPUTFORMAT
NAME "application/json"
DRIVER "OGR/GEOJSON"
MIMETYPE "application/json"
FORMATOPTION "FORM=SIMPLE"
FORMATOPTION "FILENAME=ol-query.json"
FORMATOPTION "STORAGE=memory"
FORMATOPTION "USE_FEATUREID=true"
END

Related

How to create dynamic node relation in neo4j for dynamic data?

I was able to create author nodes directly from the json file . But the challenge is on what basis or how we have to link the data. Linking "Author" to "organization". since the data is dynamic we cannot generalize it. I have tried with using csv file but, it fails the conditions when dynamic data is coming. For example one json record contain 2 organization and 3 authors, next record will be different. Different json record have different author and organization to link. organization/1 represent organization1 and organization/2 represents organization 2. Any help or hint will be great. Thank you. Please find the json file below.
"Author": [
{
"seq": "3",
"type": "abc",
"identifier": [
{
"idtype:auid": "10000000"
}
],
"familyName": "xyz",
"indexedName": "MI",
"givenName": "T",
"preferredName": {
"familyName": "xyz1",
"givenName": "a",
"initials": "T.",
"indexedName": "bT."
},
"emailAddressList": [],
"degrees": [],
"#id": "https:abc/2009127993/author/person/3",
"hasAffiliation": [
"https:abc/author/organization/1"
],
"organization": [
[
{
"identifier": [
{
"#type": "idtype:uuid",
"#subtype": "idsubtype:affiliationInstanceId",
"#value": "aff2"
},
{
"#type": "idtype:OrgDB",
"#subtype": "idsubtype:afid",
"#value": "12345"
},
{
"#type": "idtype:OrgDB",
"#subtype": "idsubtype:dptid"
}
],
"organizations": [],
"addressParts": [],
"sourceText": "",
"text": " Medical University School of Medicine",
"#id": "https:abc/author/organization/1"
}
],
[
{
"identifier": [
{
"#type": "idtype:uuid",
"#subtype": "idsubtype:affiliationInstanceId",
"#value": "aff1"
},
{
"#type": "idtype:OrgDB",
"#subtype": "idsubtype:afid",
"#value": "7890"
},
{
"#type": "idtype:OrgDB",
"#subtype": "idsubtype:dptid"
}
],
"organizations": [],
"addressParts": [],
"sourceText": "",
"text": "K University",
"#id": "https:efg/author/organization/2"
}
]
Hi I see that Organisation is part of the Author data, so you have to model it like wise. So for instance (Author)-[:AFFILIATED_WITH]->(Organisation)
When you use apoc.load.json which supports a stream of author objects you can load the data.
I did some checks on your JSON structure with this cypher query:
call apoc.load.json("file:///Users/keesv/work/check.json") yield value
unwind value as record
WITH record.Author as author
WITH author.identifier[0].`idtype:auid` as authorId,author, author.organization[0] as organizations
return authorId, author, organizations
To get this working you will need to create include apoc in the plugins directory, and add the following two lines in the apoc.conf file (create one if it is not there) in the 'conf' directory.
apoc.import.file.enabled=true
apoc.import.file.use_neo4j_config=false
I also see a nested array for the organisations in the output why is that and what is the meaning of that?
And finally I see also in the JSON that an organisation can have a reference to other organisations.
explanation
In my query I use UNWIND to unwind the base Author array. This means you get for every author a 'record' to work with.
With a MERGE or CREATE statement you can now create an Author Node with the correct properties. With the FOREACH construct you can walk over all the Organization entry and create/merge an Organization node and create the relation between the Author and the Organization.
here an 'psuedo' example
call apoc.load.json("file:///Users/keesv/work/check.json") yield value
unwind value as record
WITH record.Author as author
WITH author.identifier[0].`idtype:auid` as authorId,author, author.organization[0] as organizations
// creating the Author node
MERGE (a:Author { id: authorId })
SET a.familyName = author.familyName
...
// walk over the organizations
// determine
FOREACH (org in organizations |
MERGE (o:Organization { id: ... })
SET o.name = org.text
...
MERGE (a)-[:AFFILIATED_WITH]->(o)
// if needed you can also do a nested FOREACH here to process the Org Org relationship
)
Here is the JSON file I used I had to change something at the start and the end
[
{
"Author":{
"seq":"3",
"type":"abc",
"identifier":[
{
"idtype:auid":"10000000"
}
],
"familyName":"xyz",
"indexedName":"MI",
"givenName":"T",
"preferredName":{
"familyName":"xyz1",
"givenName":"a",
"initials":"T.",
"indexedName":"bT."
},
"emailAddressList":[
],
"degrees":[
],
"#id":"https:abc/2009127993/author/person/3",
"hasAffiliation":[
"https:abc/author/organization/1"
],
"organization":[
[
{
"identifier":[
{
"#type":"idtype:uuid",
"#subtype":"idsubtype:affiliationInstanceId",
"#value":"aff2"
},
{
"#type":"idtype:OrgDB",
"#subtype":"idsubtype:afid",
"#value":"12345"
},
{
"#type":"idtype:OrgDB",
"#subtype":"idsubtype:dptid"
}
],
"organizations":[
],
"addressParts":[
],
"sourceText":"",
"text":" Medical University School of Medicine",
"#id":"https:abc/author/organization/1"
}
],
[
{
"identifier":[
{
"#type":"idtype:uuid",
"#subtype":"idsubtype:affiliationInstanceId",
"#value":"aff1"
},
{
"#type":"idtype:OrgDB",
"#subtype":"idsubtype:afid",
"#value":"7890"
},
{
"#type":"idtype:OrgDB",
"#subtype":"idsubtype:dptid"
}
],
"organizations":[
],
"addressParts":[
],
"sourceText":"",
"text":"K University",
"#id":"https:efg/author/organization/2"
}
]
]
}
}
]
IMPORTANT create unique constraints for Author.id and Organization.id!!
In this way you can process any json file with an unknown number of author elements and an unknown number of affiliated organisations

Filtering chloropleth data using a slider for annual data in vega-lite

I have some geojson grids that map through to some annualised sales data over a period of 25 years. I am really struggling to filter this sales data by year to show the trends in a chloropleth map.
d3 = require('d3-dsv');
map_json = FileAttachment("Time_line#2.geojson").json()
sales_data = FileAttachment("Timeline_test#1.csv").csv()
vegalite = require('#observablehq/vega-lite')
vegalite ({
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"width": 600,
"height": 350,
"data": {
"name": "mapdata",
"values": map_json,
"format": {"property": "features"},
},
"params": [{
"name" : "AnnualPeriod",
"value": 1995,
"bind" : {"input": "range", "min":1995, "max":2020,"step":1 }
}],
"transform" : [{
"lookup": "properties.id",
"from": {
"data": { "values": sales_data,},
"format":"csv",
"key": "derived_boundary_id",
"fields": ["sales_volume"],
},
},],
"layer": [
{
"mark": "geoshape",
"encoding": {
"color": {
"field": "sales_volume",
"type": "quantitative",
"scale": {"scheme": "Oranges"},
},
"stroke": { "value": "#ff75"},
},
},
]
})
I have tried to add transform.filter and cannot get it to work. At the moment it appears to be taking the first sales_data record for each of the boundary_ids.
I would like the data to be filtered according to the setting of the AnnualPeriod slider.
I think I need to include something like
"transform" :[{"filter": "datum.year == AnnualPeriod"}]
I have tried it in the transform section, with the lookup between the sales_data and the geojosn objects.
I have also tried to filter in and around the geoshape mark but neither work.
Does anyone have any ideas?
The is a sample of the sales_data:
sales_volume,year,derived_boundary_id
5,2015,602212
2,2016,602212
2,2019,602212
5,1995,602213
7,1996,602213
6,1997,602213
7,1998,602213
9,1999,602213
10,2000,602213
7,2001,602213
5,2002,602213
5,2003,602213
9,2004,602213
5,2005,602213
...
where the last column maps to an "id" in the geojson data.
and this is the 'map' that I get. Always the same, irrespective of the slider setting.
I have eventually worked out how to get this to work.
Create the main data source as the sales data and attach the maps/geojson to this via a transform/lookup.
It seems so simple now, but I thought I would post the result so others can see how it can be achieved.
These observations may help others:
The transform can take the "year" filter as well as the map_json lookup.
The map/json lookup is referenced as "geo" to make it easier to understand
The geoshape mark then references, via the encoding, this geojson "geo" object.
Thanks to Mike Bostock and the Observable and Vega-Lite team for their excellent work.
vegalite ({
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"width": 600,
"height": 600,
"data": {
"values": sales_data,
},
"params": [{
"name" : "AnnualPeriod",
"value": 1995,
"bind" : {"input": "range", "min":1995, "max":2020,"step":1 }
}],
"transform" : [
{"filter" : "year(datum.year) === AnnualPeriod"},
{
"lookup": "derived_boundary_id",
"from" : {
"data": {
"values": map_json,
"format": {"property": "features"},
},
"key":"properties.id",
},
"as":"geo",
}
],
"mark":"geoshape",
"encoding": {
"shape": {
"field":"geo",
"type":"geojson",
},
"color":{
"field":"sales_volume",
"type":"quantitative",
"scale": {
"scheme":"Oranges",
"domain": [0,15],
},
}
}
})
How's your progress? I am not familiar with Choropleths, but by comparing with the Vega Examples, I spot 2 differences you may wanna take a look:
Choropleth needs a projection
Map data should be placed in lookup transform, and sale data in normal data. Ref 1 Ref 2
Do feel free to correct me if I am wrong. If you need more help, please share an editor with dummy data because I found it hard to make good use of the sale data you provided :'(

Renaming type for FSharp.Data JsonProvider

I have a JSON that looks something like this:
{
...
"names": [
{
"value": "Name",
"language": "en"
}
],
"descriptions": [
{
"value": "Sample description",
"language" "en"
}
],
...
}
When using JsonProvider from the FSharp.Data library, it maps both fields as the same type MyJsonProvider.Name. This is a little confusing when working with the code. Is there any way how to rename the type to MyJsonProvider.NameOrDescription? I have read that this is possible for the CsvProvider, but typing
JsonProvider<"./Resources/sample.json", Schema="Name->NameOrDescription">
results in an error.
Also, is it possible to define that the Description field is actually an Option<MyJsonProvider.NameOrDescription>? Or do I just have to define the JSON twice, once with all possible values and the second time just with mandatory values?
[
{
...
"names": [
{
"value": "Name",
"language": "en"
}
],
"descriptions": [
{
"value": "Sample description",
"language" "en"
}
],
...
},
{
...
"names": [
{
"value": "Name",
"language": "en"
}
],
...
}
]
To answer your first question, I do not think there is a way of specifying such renaming. It would be quite reasonable option, but the JSON provider could also be more clever when generating names here (it knows that the type can represent Name or Description, so it could generate a name with Or based on those).
As a hack, you could add an unusued field with the right name:
type A = JsonProvider<"""{
"do not use": { "value_with_langauge": {"value":"A", "language":"A"} },
"names": [ {"value":"A", "language":"A"} ],
"descriptions": [ {"value":"A", "language":"A"} ]
}""">
To answer your second question - your names and descriptions fields are already arrays, i.e. ValueWithLanguge[]. For this, you do not need an optional value. If they are not present, the provider will simply give you an empty array.

Azure Logic App: Read telemetry data as dynamic content from IoT hub message

I'm routing telemetry messages via IoT Events and event Grid to Logic Apps using a webhook. The logic app lets you input a sample JSON message and then use dynamic content to add information to an email alert I'm sending(O365: Send an Email V2)
I can include System Properties like "iothub-connection-device-id" But when I try to pick temeletry data I get the following error:
InvalidTemplate. Unable to process template language expressions in action 'Send_an_email_(V2)' inputs at line '1' and column '1680': 'The template language expression 'items('For_each')?['data']?['body']?['windingTemp1']' cannot be evaluated because property 'windingTemp1' cannot be selected. Property selection is not supported on values of type 'String'. Please see https://aka.ms/logicexpressions for usage details.'.
When I look at the raw output of the webhook connector it shows the following message but the telemetry points are cleary not there. I'd expect to see them in the "body" property but instead there is just the string: "eyJ3aW5kaW5nVGVtcDEiOjg2LjYzOTYxNzk4MjYxODMzLCJ3aW5kaW5nVGVtcDIiOjc4LjQ1MDc4NTgwMjQyMTUyLCJ3aW5kaW5nVGVtcDMiOjg1LjUzMDYxMDY5OTQ1MzY1LCJMb2FkQSI6MjAyOS44NDgyMTg4ODYxMTEsIkxvYWRCIjoyMDQwLjgxMDk4OTg0MDMzMzgsIkxvYWRWIjoyMDA0LjYxMTkzMjMyNTQ2MTgsIk9pbFRlbXAiOjk5LjA2MjMyNjU2MTY4ODU4fQ=="
Looking for help to determine what could be causing this and how to get the telemetry data passed through correctly so that I can inculde it dynamically in the email alert.
Thanks!
{
"headers": {
"Connection": "Keep-Alive",
"Accept-Encoding": "gzip,deflate",
"Host": "prod-24.northeurope.logic.azure.com",
"aeg-subscription-name": "TEMPALERT",
"aeg-delivery-count": "1",
"aeg-data-version": "",
"aeg-metadata-version": "1",
"aeg-event-type": "Notification",
"Content-Length": "1017",
"Content-Type": "application/json; charset=utf-8"
},
"body": [
{
"id": "c767fb91-3806-324c-ec3c-XXXXXXXXXX",
"topic": "/SUBSCRIPTIONS/XXXXXXXXXXXX",
"subject": "devices/Device-001",
"eventType": "Microsoft.Devices.DeviceTelemetry",
"data": {
"properties": {
"TempAlarm": "true"
},
"systemProperties": {
"iothub-connection-device-id": "Device-001",
"iothub-connection-auth-method": "{\"scope\":\"device\",\"type\":\"sas\",\"issuer\":\"iothub\",\"acceptingIpFilterRule\":null}",
"iothub-connection-auth-generation-id": "637264713410XXXX",
"iothub-enqueuedtime": "2020-06-01T23:05:58.3130000Z",
"iothub-message-source": "Telemetry"
},
"body": "eyJ3aW5kaW5nVGVtcDEiOjg2LjYzOTYxNzk4MjYxODMzLCJ3aW5kaW5nVGVtcDIiOjc4LjQ1MDc4NTgwMjQyMTUyLCJ3aW5kaW5nVGVtcDMiOjg1LjUzMDYxMDY5OTQ1MzY1LCJMb2FkQSI6MjAyOS44NDgyMTg4ODYxMTEsIkxvYWRCIjoyMDQwLjgxMDk4OTg0MDMzMzgsIkxvYWRWIjoyMDA0LjYxMTkzMjMyNTQ2MTgsIk9pbFRlbXAiOjk5LjA2MjMyNjU2MTY4ODU4fQ=="
},
"dataVersion": "",
"metadataVersion": "1",
"eventTime": "2020-06-01T23:05:58.313Z"
}
]
}
Here is the sample input I am using with the trigger:
[{
"id": "9af86784-8d40-fe2g-8b2a-bab65e106785",
"topic": "/SUBSCRIPTIONS/<subscription ID>/RESOURCEGROUPS/<resource group name>/PROVIDERS/MICROSOFT.DEVICES/IOTHUBS/<hub name>",
"subject": "devices/LogicAppTestDevice",
"eventType": "Microsoft.Devices.DeviceTelemetry",
"eventTime": "2019-01-07T20:58:30.48Z",
"data": {
"body": {
"windingTemp1": 95.62818310718433
},
"properties": {
"Status": "Active"
},
"systemProperties": {
"iothub-content-type": "application/json",
"iothub-content-encoding": "utf-8",
"iothub-connection-device-id": "d1",
"iothub-connection-auth-method": "{\"scope\":\"device\",\"type\":\"sas\",\"issuer\":\"iothub\",\"acceptingIpFilterRule\":null}",
"iothub-connection-auth-generation-id": "123455432199234570",
"iothub-enqueuedtime": "2019-01-07T20:58:30.48Z",
"iothub-message-source": "Telemetry"
}
},
"dataVersion": "",
"metadataVersion": "1"
}]
Summary comment to answer to help others who have same problem.
The body you provided is Base64 encoded, you can decode it with Convert.FromBase64String(String) Method.
byte[] newBytes = Convert.FromBase64String(body);
For more details, you could refer to this issue.
Update:
Add the following code in my application will solve the problem.
message.ContentEncoding = "utf-8";
message.ContentType = "application/json";

Breeze.js - nonscalar complex properties cause circular structure exception during em.exportEntities

This issue, with scalar complex properties, was reported earlier and resolved in breeze 1.3.5.
I am still seeing it, with non-scalar complex properties, in breeze 1.4.5. After creating an entity using this metadata, the exportEntities() method on the entity manager fails with an exception in JSON.stringify, complaining about a circular reference.
Here's some code to replicate the problem:
var jsonMetadata = {
"metadataVersion": "1.0.5",
"namingConvention": "camelCase",
"localQueryComparisonOptions": "caseInsensitiveSQL",
"dataServices": [{"serviceName": "breeze/myservice/" } ],
"structuralTypes": [
{
"shortName": "Address",
"namespace": "mynamespace",
"isComplexType": true,
"dataProperties": [
{"name": "street"},
{"name": "city"},
]
},
{
"shortName": "Person",
"namespace": "mynamespace",
"autoGeneratedKeyType": "Identity",
"defaultResourceName": "Person",
"dataProperties": [
{"name": "_id", "dataType": "MongoObjectId", "isNullable": false, "defaultValue": "",
"isPartOfKey": true },
{"name": "displayName", "dataType": "String"},
{ "name": "addresses",
"complexTypeName": "Address:#mynamespace",
"isScalar": false
}
]
}
],
"resourceEntityTypeMap": {
"Person": "Person:#mynamespace"
}};
var manager = new breeze.EntityManager();
manager.metadataStore.importMetadata(jsonMetadata);
var person = manager.createEntity('Person', {displayName: "Joe Bob"});
var myAddresses = person.getProperty('addresses');
var myAddressProp = manager.metadataStore.getEntityType("Address").createInstance(
{street: "Main", city:"Pleasantville"});
myAddresses.push(myAddressProp);
console.log("Complex property is a circular datatype, cannot convert to JSON - that's fine")
//JSON.stringify(person.addresses); // fails with error
console.log("... except that manager.exportEntities() doesn't handle that case!");
var entities = manager.exportEntities(); // also fails
The circular reference that JSON.stringify is complaining about seems to be in the 'parent' property of the ComplexAspect of the Address property.
Also, if there's a simpler way to populate the addresses array, I'd appreciate some help.
Ok, this should be fixed as of Breeze v 1.4.6 ( or later) available now
------------- Original Post ------------------
This is a bug. It will be fixed in the next release, out later this week or early next week. and... thanks for the repro. I will post back when it gets in.

Resources