Related
I am facing issues with elasticsearch aggregation grouping inside top_hits. or i need unique students count in the tophits
Elastic search mapping:
{
"board" : {
"properties" : {
"notApplied" : {
"type" : "date"
}
}
}
}
Query :
{
"size": 0,
"query": {},
"aggs": {
"notApplied": {
"filter": {
"exists": {
"field": "board.notApplied"
}
},
"aggs": {
"top_student_hits": {
"top_hits": {
"sort": [
{
"board.notApplied": {
"order": "desc"
}
}
],
"script_fields": {
"dues": {
"script": {
"source": "if (doc.containsKey('board.notApplied') && doc['board.notApplied'].size() != 0) { (doc['board.notApplied'].value.toInstant().toEpochMilli()-params.date)/86400000 } else { 0; }",
"params": {
"date": 1669939199059 // --> < 1 day
}
}
}
},
"_source": {
"includes": [
"id",
"studentName",
"usercode",
"board.notApplied",
"userId"
]
},
"size": 5
}
}
}
}
}
}
Output for the above query :
{
"took" : 11,
...
"aggregations" : {
"notApplied" : {
"doc_count" : 42,
"top_student_hits" : {
"hits" : {
"total" : {
"value" : 42,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : "applications",
"_type" : "_doc",
"_id" : "4b85533822f91e9b99392f16dedaae1f",
"_score" : null,
"_source" : {
"board" : {
"notApplied" : "2022-10-25T00:00:00.000Z"
},
"studentName" : "Joe",
"id" : "4b85533822f91e9b99392f16dedaae1f",
"userId" : "45a47d1314041ab287a277679ff19922"
},
"fields" : {
"dues" : [
-37
]
},
"sort" : [
1666656000000
]
},
{
"_index" : "applications",
"_type" : "_doc",
"_id" : "1897f32d2d7f691e42c3fe6ebe631c7d",
"_score" : null,
"_source" : {
"board" : {
"notApplied" : "2022-10-25T00:00:00.000Z"
},
"studentName" : "Joe",
"id" : "1897f32d2d7f691e42c3fe6ebe631c7d",
"userId" : "45a47d1314041ab287a277679ff19922"
},
"fields" : {
"dues" : [
-37
]
},
"sort" : [
1666656000000
]
},
{
"_index" : "applications",
"_type" : "_doc",
"_id" : "f0b25dc9a911782ace5af36db7bfbc1f",
"_score" : null,
"_source" : {
"board" : {
"notApplied" : "2022-10-25T00:00:00.000Z"
},
"studentName" : "Sam",
"id" : "f0b25dc9a911782ace5af36db7bfbc1f",
"userId" : "d84f9e5231daa902c37921de9126cad7"
},
"fields" : {
"dues" : [
-37
]
},
"sort" : [
1666656000000
]
},
{
"_index" : "applications",
"_type" : "_doc",
"_id" : "e7f84fa978a553e77716ab479d3d6ce5",
"_score" : null,
"_source" : {
"board" : {
"notApplied" : "2022-10-13T00:00:00.000Z"
},
"id" : "e7f84fa978a553e77716ab479d3d6ce5",
"studentName" : "Sam",
"userId" : "d84f9e5231daa902c37921de9126cad7"
},
"fields" : {
"dues" : [
-49
]
},
"sort" : [
1665619200000
]
},
{
"_index" : "applications",
"_type" : "_doc",
"_id" : "9cba9f6b0d7a28ef739b321291d00170",
"_score" : null,
"_source" : {
"board" : {
"notApplied" : "2022-09-20T00:00:00.000Z"
},
"studentName" : "Ctest17 ",
"id" : "9cba9f6b0d7a28ef739b321291d00170",
"userId" : "ddaf6d6162c8317fd90fec0b870132ce"
},
"fields" : {
"dues" : [
-72
]
},
"sort" : [
1663632000000
]
}
]
}
}
}
}
}
I am getting the exact results but it has been duplicated by userId.
i need a result in top_hits without duplicates or the buckets should be grouped by userId. also the result should be sort desc by (dues or notApplied) field.
can any one help me to resolve this?
Swagger/OpenAPI definition:
{
"openapi" : "3.0.1",
"info" : {
"title" : "OpenAPI definition",
"version" : "v0"
},
"servers" : [ {
"url" : "http://sandbox.test.com:8063/api/recs",
"description" : "Generated server url"
} ],
"paths" : {
"/data" : {
"get" : {
"tags" : [ "Data" ],
"operationId" : "getData",
"parameters" : [ {
"name" : "goal",
"in" : "query",
"required" : false,
"schema" : {
"$ref" : "#/components/schemas/GoalsEnum_User"
}
} ],
"responses" : {
"404" : {
"description" : "Not Found",
"content" : {
"*/*" : {
"schema" : {
"type" : "object"
}
}
}
},
"200" : {
"description" : "Result generated successfully",
"content" : {
"application/json" : {
"schema" : {
"type" : "array",
"items" : {
"oneOf" : [ {
"$ref" : "#/components/schemas/EventDataDto"
}, {
"$ref" : "#/components/schemas/FreeRideDataDto"
}]
}
}
}
}
}
}
}
}
},
"components" : {
"schemas" : {
"GoalsEnum_User" : {
"type" : "string",
"enum" : [ "User1", "User2" ]
},
"EventDataDto" : {
"type" : "object",
"allOf" : [ {
"$ref" : "#/components/schemas/ParentDataSchema_UserData"
}, {
"type" : "object",
"properties" : {
"rules" : {
"type" : "array",
"items" : {
"$ref" : "#/components/schemas/RuleDto"
}
}
}
}, {
"$ref" : "#/components/schemas/ParentDataSchema"
} ]
},
"FreeRideDataDto" : {
"type" : "object",
"allOf" : [ {
"$ref" : "#/components/schemas/ParentDataSchema"
}, {
"type" : "object",
"properties" : {
"completedRoutes" : {
"type" : "array",
"items" : {
"type" : "integer",
"format" : "int64"
}
},
"averageDistance" : {
"type" : "number",
"format" : "double"
},
"averageDuration" : {
"type" : "number",
"format" : "double"
}
}
}, {
"$ref" : "#/components/schemas/ParentDataSchema_UserData"
} ]
},
"ParentDataSchema" : {
"required" : [ "type" ],
"type" : "object",
"properties" : {
"type" : {
"type" : "string",
"enum" : [ "FREE_RIDE", "EVENT" ]
}
},
"discriminator" : {
"propertyName" : "type",
"mapping" : {
"EVENT" : "#/components/schemas/EventRecommendationDto",
"FREE_RIDE" : "#/components/schemas/FreeRideRecommendationDto"
}
}
},
"ParentDataSchema_UserData" : {
"required" : [ "type" ],
"type" : "object",
"properties" : {
"type" : {
"type" : "string",
"enum" : [ "FREE_RIDE", "EVENT" ]
}
},
"discriminator" : {
"propertyName" : "type",
"mapping" : {
"EVENT" : "#/components/schemas/EventRecommendationDto",
"FREE_RIDE" : "#/components/schemas/FreeRideRecommendationDto"
}
}
}
}
}
}
Generated Example:
[
{
"type": "FREE_RIDE",
"rules": [
{
"ruleId": 0,
"categoryId": 0,
"name": "string",
"type": "string",
"value": "string"
}
]
},
{
"type": "FREE_RIDE",
"completedRoutes": [
0
],
"averageDistance": 0,
"averageDuration": 0
}
]
Since there are specific values for the discriminating field "type", I expect the examples to have the correct value for detected types. Although the types were listed correctly, the type field is not set to the discriminating value.
Is there anything I can do to the Swagger/OpenAPI definitions or Swagger UI to fix this? I'm even open to adding a bug-fix if you can point me to where the values of the field examples are set and how can I choose the discriminating value instead of the first one in the enum instead.
I have 2 indexes products & shop_inventory_6(shop wise inventory)
products mapping
{
"products_staging" : {
"aliases" : { },
"mappings" : {
"product" : {
"properties" : {
"alternate_names" : {
"type" : "text"
},
"brand" : {
"properties" : {
"id" : {
"type" : "integer"
},
"image_url" : {
"type" : "text",
"index" : false
},
"name" : {
"type" : "text",
"analyzer" : "standard"
}
}
},
"brand_suggest" : {
"type" : "completion",
"analyzer" : "autocomplete",
"search_analyzer" : "whitespace_analyzer",
"preserve_separators" : true,
"preserve_position_increments" : true,
"max_input_length" : 50
},
"category" : {
"properties" : {
"id" : {
"type" : "integer"
},
"image_url" : {
"type" : "text",
"index" : false
},
"name" : {
"type" : "text",
"analyzer" : "standard"
}
}
},
"id" : {
"type" : "text"
},
"image_url" : {
"type" : "text",
"index" : false
},
"name" : {
"type" : "text",
"fields" : {
"raw" : {
"type" : "keyword"
}
},
"analyzer" : "standard"
},
"name_suggest" : {
"type" : "completion",
"analyzer" : "autocomplete",
"search_analyzer" : "whitespace_analyzer",
"preserve_separators" : true,
"preserve_position_increments" : true,
"max_input_length" : 50
},
"product_alternate_name_suggest" : {
"type" : "completion",
"analyzer" : "autocomplete",
"search_analyzer" : "whitespace_analyzer",
"preserve_separators" : true,
"preserve_position_increments" : true,
"max_input_length" : 50
},
"product_sizes" : {
"type" : "nested",
"properties" : {
"ean_code" : {
"type" : "keyword"
},
"id" : {
"type" : "integer"
},
"is_deleted" : {
"type" : "boolean"
},
"price" : {
"type" : "float"
},
"shop_category_type_ids" : {
"type" : "text"
},
"uom" : {
"type" : "keyword"
},
"weight" : {
"type" : "float"
}
}
},
"sub_category" : {
"properties" : {
"alternate_names" : {
"type" : "text"
},
"id" : {
"type" : "integer"
},
"image_url" : {
"type" : "text",
"index" : false
},
"name" : {
"type" : "text",
"analyzer" : "standard"
}
}
},
"sub_category_alternate_suggest" : {
"type" : "completion",
"analyzer" : "autocomplete",
"search_analyzer" : "whitespace_analyzer",
"preserve_separators" : true,
"preserve_position_increments" : true,
"max_input_length" : 50
},
"sub_category_suggest" : {
"type" : "completion",
"analyzer" : "autocomplete",
"search_analyzer" : "whitespace_analyzer",
"preserve_separators" : true,
"preserve_position_increments" : true,
"max_input_length" : 50
}
}
}
},
"settings" : {
"index" : {
"number_of_shards" : "3",
"provided_name" : "products_staging",
"creation_date" : "1566968865962",
"analysis" : {
"filter" : {
"autocomplete_filter" : {
"type" : "edge_ngram",
"min_gram" : "2",
"max_gram" : "20"
}
},
"analyzer" : {
"autocomplete" : {
"filter" : [
"lowercase",
"autocomplete_filter"
],
"type" : "custom",
"tokenizer" : "standard"
},
"whitespace_analyzer" : {
"filter" : [
"lowercase",
"asciifolding"
],
"type" : "custom",
"tokenizer" : "whitespace"
}
}
},
"number_of_replicas" : "1",
"uuid" : "M5GE3TK9QOKVaBMcOkCJPQ",
"version" : {
"created" : "6000199"
}
}
}
}
}
shop_inventory mapping
{
"staging_shop_inventory_17" : {
"aliases" : { },
"mappings" : {
"shop_inventory" : {
"properties" : {
"brand" : {
"properties" : {
"created_at" : {
"type" : "date"
},
"id" : {
"type" : "integer"
},
"image" : {
"type" : "text",
"index" : false
},
"is_selected" : {
"type" : "boolean"
},
"name" : {
"type" : "text",
"analyzer" : "standard"
},
"updated_at" : {
"type" : "date"
}
}
},
"brand_suggest" : {
"type" : "text",
"analyzer" : "ngram_analyzer"
},
"category" : {
"properties" : {
"id" : {
"type" : "integer"
},
"image" : {
"type" : "text",
"index" : false
},
"name" : {
"type" : "text",
"analyzer" : "standard"
}
}
},
"deleted_at" : {
"type" : "date"
},
"id" : {
"type" : "integer"
},
"image" : {
"type" : "text",
"index" : false
},
"is_deleted" : {
"type" : "boolean"
},
"name" : {
"type" : "text",
"fields" : {
"raw" : {
"type" : "keyword"
}
},
"analyzer" : "gramAnalyzer",
"search_analyzer" : "whitespace_analyzer"
},
"name_suggest" : {
"type" : "text",
"analyzer" : "ngram_analyzer"
},
"product_deleted" : {
"type" : "keyword"
},
"product_id" : {
"type" : "integer"
},
"product_sizes" : {
"type" : "nested",
"properties" : {
"deleted_at" : {
"type" : "date"
},
"ean_code" : {
"type" : "keyword"
},
"id" : {
"type" : "integer"
},
"in_stock" : {
"type" : "boolean"
},
"is_deleted" : {
"type" : "boolean"
},
"price" : {
"type" : "float"
},
"product_update_on" : {
"type" : "date"
},
"product_update_status" : {
"type" : "integer"
},
"uom" : {
"type" : "keyword"
},
"weight" : {
"type" : "float"
}
}
},
"sub_category" : {
"properties" : {
"created_at" : {
"type" : "date"
},
"id" : {
"type" : "integer"
},
"image" : {
"type" : "text",
"index" : false
},
"is_selected" : {
"type" : "boolean"
},
"name" : {
"type" : "text",
"analyzer" : "standard"
},
"updated_at" : {
"type" : "date"
}
}
},
"sub_category_suggest" : {
"type" : "text",
"analyzer" : "gramAnalyzer",
"search_analyzer" : "whitespace_analyzer"
}
}
}
},
"settings" : {
"index" : {
"number_of_shards" : "3",
"provided_name" : "staging_shop_inventory_17",
"creation_date" : "1569230448054",
"analysis" : {
"filter" : {
"gramFilter" : {
"token_chars" : [
"letter",
"digit"
],
"min_gram" : "1",
"type" : "edge_ngram",
"max_gram" : "20"
}
},
"analyzer" : {
"whitespace_analyzer" : {
"filter" : [
"lowercase",
"asciifolding"
],
"type" : "custom",
"tokenizer" : "whitespace"
},
"ngram_analyzer" : {
"token_chars" : [
"letter",
"digit"
],
"min_gram" : "4",
"type" : "custom",
"max_gram" : "20",
"tokenizer" : "ngram"
},
"gramAnalyzer" : {
"filter" : [
"lowercase",
"asciifolding",
"gramFilter"
],
"type" : "custom",
"tokenizer" : "whitespace"
}
}
},
"number_of_replicas" : "1",
"uuid" : "q9BkwXMVQnGoga8tznNFgg",
"version" : {
"created" : "6000199"
}
}
}
}
}
I want to select products from products index which are not in shop_inventory index. without two queries
Also I want select product by sub_category_id & brand_ids where I have multiple sub_category_id & brand ids (because my brand belongs to multiple categories) without using OR condition
I am struggling with elasticsearch-rails.
I have the following mapping:
{
"listings" : {
"mappings" : {
"listing" : {
"properties" : {
"address" : {
"type" : "string"
},
"authorized" : {
"type" : "boolean"
},
"categories" : {
"properties" : {
"created_at" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"id" : {
"type" : "long"
},
"name" : {
"type" : "string"
},
"parent_id" : {
"type" : "long"
},
"updated_at" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"url_name" : {
"type" : "string"
}
}
},
"cid" : {
"type" : "string"
},
"city" : {
"type" : "string"
},
"country" : {
"type" : "string"
},
"created_at" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"featured" : {
"type" : "boolean"
},
"geojson" : {
"type" : "string"
},
"id" : {
"type" : "long"
},
"latitude" : {
"type" : "string"
},
"longitude" : {
"type" : "string"
},
"name" : {
"type" : "string"
},
"phone" : {
"type" : "string"
},
"postal" : {
"type" : "string"
},
"province" : {
"type" : "string"
},
"thumbnail_filename" : {
"type" : "string"
},
"updated_at" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"url" : {
"type" : "string"
}
}
}
}
}
}
I would like to change the type for the geojson field from string to geo_point so I can use the geo_shape query on it.
I tried this in my model:
settings index: { number_of_shards: 1 } do
mappings dynamic: 'false' do
indexes :geojson, type: 'geo_shape'
end
end
with peculiar results. When I queried the mapping with $ curl 'localhost:9200/_all/_mapping?pretty', the geojson field still shows as type: string.
Within a Rails console, if I do Listing.mappings.to_hash, it seems to show that the geojson field is of type geo_shape.
And yet when running this query:
Listing.search(query: { fuzzy_like_this: { fields: [:name], like_text: "gap" } }, query: { fuzzy_like_this_field: { city: { like_text: "San Francisco" } } }, query: { geo_shape: { geojson: { shape: { type: :envelope, coordinates: [[37, -122],[38,-123]] } } } }); response.results.total; response.results.map { |r| puts "#{r._score} | #{r.name}, #{r.city} (lat: #{r.latitude}, lon: #{r.longitude})" }
ES complains that the geojson field is not of type geo_shape.
What am I missing? How do I tell ES that I want the geojson field to be of type geo_shape and not string?
The issue was that I didn't delete and recreate the mapping.
In the rails console, I ran Model.__elasticsearch__.delete_index! and then Model.__elasticsearch__.create_index! followed by Model.import
searching using elastic search gives me following error on console
{"_shards":{"total":5,"successful":0,"failed":5,"failures":[{"index":"test","shard":4,"reason":"BroadcastShardOperationFailedException[[test][4] ]; nested: ElasticSearchException[failed to execute suggest]; nested: ElasticSearchIllegalArgumentException[[suggest] does not support [size]]; "},{"index":"test","shard":3,"reason":"BroadcastShardOperationFailedException[[test][3] ]; nested: ElasticSearchException[failed to execute suggest]; nested: ElasticSearchIllegalArgumentException[[suggest] does not support [size]]; "},{"index":"test","shard":2,"reason":"BroadcastShardOperationFailedException[[test][2] ]; nested: ElasticSearchException[failed to execute suggest]; nested: ElasticSearchIllegalArgumentException[[suggest] does not support [size]]; "},{"index":"test","shard":0,"reason":"BroadcastShardOperationFailedException[[test][0] ]; nested: ElasticSearchException[failed to execute suggest]; nested: ElasticSearchIllegalArgumentException[[suggest] does not support [size]]; "},{"index":"test","shard":1,"reason":"BroadcastShardOperationFailedException[[test][1] ]; nested: ElasticSearchException[failed to execute suggest]; nested: ElasticSearchIllegalArgumentException[[suggest] does not support [size]]; "}]}}
not able to understand probable reason for this i am using tire/karmi for this
the ES version is 0.9.9
this is my mapping code
{
"test" : {
"document" : {
"properties" : {
"artikelnummer" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs"
},
"bestelltext" : {
"type" : "multi_field",
"fields" : {
"bestelltext" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"bezeichnung" : {
"type" : "multi_field",
"fields" : {
"bezeichnung" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
},
"suggest" : {
"type" : "string",
"analyzer" : "suggest_analyzer",
"include_in_all" : false
}
}
},
"bezeichnung_zusatz" : {
"type" : "multi_field",
"fields" : {
"bezeichnung_zusatz" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"dimension" : {
"type" : "string",
"include_in_all" : false
},
"eannummer" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs"
},
"gewicht" : {
"type" : "float",
"include_in_all" : false
},
"gruppe" : {
"type" : "multi_field",
"fields" : {
"gruppe" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"gruppe_nummer" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs",
"include_in_all" : false
},
"gruppe_zusatz" : {
"type" : "multi_field",
"fields" : {
"gruppe_zusatz" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"hersteller" : {
"type" : "multi_field",
"fields" : {
"hersteller" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
},
"unchanged" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs",
"include_in_all" : false
}
}
},
"hersteller_artikelnummer" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs"
},
"hersteller_nummer" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs",
"include_in_all" : false
},
"hierarchie" : {
"type" : "multi_field",
"fields" : {
"hierarchie" : {
"type" : "string",
"index_analyzer" : "hierarchie_index_analyzer",
"search_analyzer" : "keyword",
"include_in_all" : false
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"infotext" : {
"type" : "multi_field",
"fields" : {
"infotext" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"langtext" : {
"type" : "multi_field",
"fields" : {
"langtext" : {
"type" : "string"
},
"ngram" : {
"type" : "string",
"index_analyzer" : "ngram_index_analyzer",
"search_analyzer" : "ngram_search_analyzer",
"include_in_all" : false
}
}
},
"listenpreis_brutto" : {
"type" : "float",
"include_in_all" : false
},
"listenpreis_netto" : {
"type" : "float",
"include_in_all" : false
},
"matchcode" : {
"type" : "string",
"index" : "not_analyzed",
"omit_norms" : true,
"index_options" : "docs"
},
"mengeneinheit" : {
"type" : "string",
"include_in_all" : false
}
}
}
}
}
This has got nothing to do with your mapping. It's to do with the search call you're making. Somewhere you'll have the "size" keywords within your call to the search API.
It looks like you're using the suggester endpoint and you've got the "size" keyword directly under the "suggest" keyword. Here's an example of my PHP code (I know it's not in Ruby but it still illustrates the point).
$searchParams['body']['suggest'] = array(
'text' => strtolower(trim(urldecode($query))),
'simple_phrase' => array(
'phrase' => array(
'field' => "name",
'size' => 4,
'real_word_error_likelihood' => 0.95,
'confidence' => 1.0,
'gram_size' => 1,
'direct_generator' => array(
array(
"field" => "name",
"suggest_mode" => "always",
"min_word_len" => 1
),
array(
"field" => "name_reverse",
"suggest_mode" => "always",
"min_word_len" => 1,
"pre_filter" => "reverse",
"post_filter" => "reverse"
)
)
)
)
);
You see how I have the "size" keyword under the phrase suggester, rather than under suggester directly.