Elasticsearch dynamic template to match several exact fields - templates

I'm currently struggling to simplify my mapping template files for Elasticsearch. Indeed, I got several Object fields that have the same structure (e.g. source and destination here)
Is there a way to set up Dynamic template so that it can match several patterns ?
Here's what I execute:
POST /_template/mapping-lol
{
"template": "*-newevents-*",
"mappings": {
"log": {
"dynamic_templates": [
{
"system": {
"match_pattern": "regex",
"match": "^(source|destination)$",
"mapping": {
"properties": {
"name": {
"dynamic": false,
"type": "object",
"properties": {
"first": {
"type": "text"
},
"last": {
"type": "text"
}
}
},
"ip": {
"type": "ip"
}
}
}
}
}
],
"properties": {
"source": {
"type": "object",
"dynamic": true
},
"destination": {
"type": "object",
"dynamic": true
}
}
}
}
}
POST /tenant-newevents-1/log
{
"source": {
"name": {
"first": "John",
"last": "Doe"
},
"ip": "1.2.3.4"
},
"destination": {
"name": {
"first": "Jane",
"last": "Doe"
},
"ip": "3.4.5.6"
}
}
GET /tenant-newevents-1
This above does not work...
I've got plenty of these same schemes to match (~20).
Thank you very much for your help !

OK I found out what went wrong: Fields mustn't be mapped at all for Dynamic mapping to proceed. Removing "source" and "destination" schemes in mapping worked.
POST /_template/mapping-lol
{
"template": "*-newevents-*",
"mappings": {
"log": {
"dynamic_templates": [
{
"system": {
"match_pattern": "regex",
"match": "^(source|destination)$",
"mapping": {
"properties": {
"name": {
"dynamic": false,
"type": "object",
"properties": {
"first": {
"type": "text"
},
"last": {
"type": "text"
}
}
},
"ip": {
"type": "ip"
}
}
}
}
}
],
"properties": {}
}
}
}

Related

Infinite loading using Authorization header with swagger 2.0

I'm programming a swagger documentation with swagger 2.0 and the request containing an authorization header doesn't seem to work properly. In fact, when I add the token in the Authorize header then execute the query, it says loading indefinitely.
Loading request
I've been facing the same problem for 2 days and I don't find any topic dealing about my issue.
{
"swagger": "2.0",
"info": {
"description": "Swagger API",
"version": "1.0.0",
"title": "Swagger API",
"license": {
"name": "MIT",
"url": "https://opensource.org/licenses/MIT"
}
},
"securityDefinitions": {
"Bearer": {
"type": "apiKey",
"name": "Authorization",
"in": "header"
}
},
"paths": {
"/api/login": {
"post": {
"tags": ["Login"],
"summary": "Returns JWT",
"parameters": [
{
"in": "body",
"name": "Login body",
"description": "Login request used to obtain JWT",
"required": true,
"schema": {
"$ref": "#/components/Login"
}
}
],
"responses": {
"200": {
"description": "Success"
}
}
}
},
"/api/devices": {
"get": {
"tags": ["Devices"],
"summary": "Returns all devices",
"security": {
"Bearer": []
},
"responses": {
"200": {
"description": "GET success"
},
"401": {
"description": "Missing header with jwt"
}
}
},
"post": {
"tags": ["Devices"],
"summary": "Deploy all devices",
"security": {
"Bearer": []
},
"parameters": [
{
"in": "body",
"name": "Devices POST body",
"description": "Deploy devices",
"required": true,
"schema": {
"$ref": "#/components/Devices"
}
}
],
"responses": {
"200": {
"description": "Device deployment succeed"
},
"401": {
"description": "Missing header with jwt"
}
}
}
}
},
"components": {
"Login": {
"type": "object",
"properties": {
"login": {
"type": "object",
"properties": {
"email": {
"type": "string"
},
"password": {
"type": "string"
}
}
}
}
},
"Devices": {
"type": "object",
"properties": {
"devices": {
"type": "object",
"properties": {
"devEUILSBList": {
"type": "array",
"items": {
"type": "string"
}
},
"applicationID": {
"type": "integer"
},
"deviceProfileID": {
"type": "string"
}
}
}
}
}
}
}
Complementary information :
The backend is running with Flask
Swagger 2.0

How to validate nested properties in component schema (openapi 3 in Postman)

I'm working on an OpenAPI 3 schema.
I would like to use a data model from the components.schemas inside the responses content and have some required nested properties inside that data model. However, it doesn't seem like the required validation is being applied. I'm testing this in Postman with a mock server.
Here is my schema:
{
"openapi": "3.0.0",
"info": {
"version": "1.0.0",
"title": "Usage stats API"
},
"servers": [
{
"url": "http://some-middleware-endpoint.com"
}
],
"paths": {
"/publishers/{publisherId}/files/{fileId}": {
"get": {
"summary": "Get single file for publisher",
"parameters": [
{
"name": "publisherId",
"in": "path",
"description": "ID of the publisher",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
}
},
{
"name": "fileId",
"in": "path",
"description": "ID of the file",
"required": true,
"schema": {
"type": "integer",
"format": "int64"
}
}
],
"responses": {
"200": {
"description": "File for publisher",
"headers": {
"Content-Type": {
"description": "application/json"
}
},
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"meta"
],
"properties": {
"meta": {
"type": "object",
"required": ["page"],
"properties": {
"$ref": "#/components/schemas/Pagination"
}
}
}
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"Pagination": {
"properties": {
"page": {
"required": ["current-page", "per-page", "from", "to", "total", "last-page"],
"type": "object",
"properties": {
"current-page": {
"type": "integer"
},
"per-page": {
"type": "integer"
},
"from": {
"type": "integer"
},
"to": {
"type": "integer"
},
"total": {
"type": "integer"
},
"last-page": {
"type": "integer"
}
}
}
}
}
}
}
}
This response passes validation:
{
"meta": {
"page": {}
}
}
Even though all of the attributes I've required ("required": ["current-page", "per-page", "from", "to", "total", "last-page"]) are not present.
Basically, I would like page and all its nested properties to be required.
I guess I'm doing something wrong in defining the properties. Any help is appreciated!
Oh well, I guess my issue was pulling up the $ref one level up.
The following seems to work inside responses.content.
"meta": {
"type": "object",
"required": [
"page"
],
"$ref": "#/components/schemas/Pagination"
}
instead of
"meta": {
"type": "object",
"required": ["page"],
"properties": {
"$ref": "#/components/schemas/Pagination"
}
}

Logstash keep creating field despite the dynamic_mapping being deactivated

I have defined my own template to be used by logstash where I have deactivate the dynamic mapping:
{
"my_index": {
"order": 0,
"template": "my_index",
"settings": {
"index": {
"mapper": {
"dynamic": "false"
},
"analysis": {
"analyzer": {
"nlp_analyzer": {
"filter": [
"lowercase"
],
"type": "custom",
"tokenizer": "nlp_tokenizer"
}
},
"tokenizer": {
"nlp_tokenizer": {
"pattern": ""
"(\w+)|(\s*[\s+])"
"",
"type": "pattern"
}
}
},
"number_of_shards": "1",
"number_of_replicas": "0"
}
},
"mappings": {
"author": {
"properties": {
"author_name": {
"type": "keyword"
},
"author_pseudo": {
"type": "keyword"
},
"author_location": {
"type": "text",
"fields": {
"standard": {
"analyzer": "standard",
"term_vector": "yes",
"type": "text"
},
"nlp": {
"analyzer": "nlp_analyzer",
"term_vector": "yes",
"type": "text"
}
}
}
}
}
}
}
}
To test if elasticsearch won’t generate new field I try to let a field in my events that is not present in my mapping, let’s say that I have this event:
{
“type” => “author”,
“author_pseudo” => “chloemdelorenzo”,
“author_name” => “Chloe DeLorenzo”,
“author_location” => “US”,
}
Elasticsearch will generate a new field in the mapping when indexing this event:
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
I know that Logstash is using my template because in my mapping I use a custom analyser and I can find it back into the mapping generated. But apparently it doesn’t take into consideration that the dynamic field is disabled.
I want elasticsearch to ignore fields that are not present in my mapping but to index the field that have a defined mapping. How can I avoid logstash to create new field?
You should enforce the mapping at the document type level.
https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-mapping.html
Regardless of the value of this setting, types can still be added
explicitly when creating an index or with the PUT mapping API.
So your mapping will look like:
"mappings": {
"author": {
"dynamic": false,
"properties": {
"author_name": {
"type": "keyword"
},
"author_pseudo": {
"type": "keyword"
},
"author_location": {
"type": "text",
"fields": {
"standard": {
"analyzer": "standard",
"term_vector": "yes",
"type": "text"
},
"nlp": {
"analyzer": "nlp_analyzer",
"term_vector": "yes",
"type": "text"
}
}
}
}
}
}
This answer is not exactly what you are requesting, but you can manually remove fields with a logstash filter like this:
filter {
mutate {
remove_field => ["fieldname"]
}
}
If your events have a defined list of fields, you could solve your problem this way.

Aggregation by a compound field (copy_to) not working on Elasticsearch

I have an index in Elasticsearch (v 1.5.0) that has a mapping that looks like this:
{
"storedash": {
"mappings": {
"outofstock": {
"_ttl": {
"enabled": true,
"default": 1296000000
},
"properties": {
"CompositeSKUProductId": {
"type": "string"
},
"Hosts": {
"type": "nested",
"properties": {
"HostName": {
"type": "string"
},
"SKUs": {
"type": "nested",
"properties": {
"CompositeSKUProductId": {
"type": "string",
"index": "not_analyzed"
},
"Count": {
"type": "long"
},
"ProductId": {
"type": "string",
"index": "not_analyzed",
"copy_to": [
"CompositeSKUProductId"
]
},
"SKU": {
"type": "string",
"index": "not_analyzed",
"copy_to": [
"CompositeSKUProductId"
]
}
}
}
}
},
"Timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
}
}
Look at how field CompositeSKUProductId is created as a composition of both the SKU and ProductId fields.
I now want to perform an aggregation on that composite field, but it doesn't seem to work; the relevant part of my query looks like this:
"aggs": {
"hostEspecifico": {
"filter": {
"term": { "Hosts.HostName": "www.example.com"}
},
"aggs": {
"skus": {
"nested": {
"path": "Hosts.SKUs"
},
"aggs": {
"valores": {
"terms": {
"field": "Hosts.SKUs.CompositeSKUProductId", "order": { "media": "desc" }, "size": 100 },
"aggs": {
"media": {
"avg": {
"field": "Hosts.SKUs.Count"
}
}
}
}
}
}
}
}
}
Thing is, this aggregation returned zero buckets, as though it weren't even there.
I checked that the very same query works if only I change CompositeSKUProductId by another field like ProductId.
Any ideas as to what I can do to solve my problem?
N.B.: I'm using the AWS Elasticsearch Service, which does not allow scripting.
The problem here is that you have misunderstood the concept of copy_to functionality. It simply copies the field values of various fields and does not combine the way you would expect.
If SKU is 123 and product id is 456 then composite field will have them as separate values and not 123 456. You can verify this by querying your field.
You would have to do this on server side, ideally with script but it is not allowed. Personally we used AWS ES service but faced multiple problems, major being not able to change elasticsearch.yml file and not able to use scripts. You might want to look at Found.
Hope this helps!
In order to copy_to another field in the nested doc, you need to supply the full path to the field you want to copy to in your mapping. You have only provided "CompositeSKUProductId", which causes the data to be copied to a field in your root document, instead of your nested SKUs type document.
Try updating your mapping for your "SKUs" type to copy_to the fully qualified field "Hosts.SKUs.CompositeSKUProductId" instead.
Like this:
{
"storedash": {
"mappings": {
"outofstock": {
"_ttl": {
"enabled": true,
"default": 1296000000
},
"properties": {
"CompositeSKUProductId": {
"type": "string"
},
"Hosts": {
"type": "nested",
"properties": {
"HostName": {
"type": "string"
},
"SKUs": {
"type": "nested",
"properties": {
"CompositeSKUProductId": {
"type": "string",
"index": "not_analyzed"
},
"Count": {
"type": "long"
},
"ProductId": {
"type": "string",
"index": "not_analyzed",
"copy_to": [
"Hosts.SKUs.CompositeSKUProductId"
]
},
"SKU": {
"type": "string",
"index": "not_analyzed",
"copy_to": [
"Hosts.SKUs.CompositeSKUProductId"
]
}
}
}
}
},
"Timestamp": {
"type": "date",
"format": "dateOptionalTime"
}
}
}
}
}
}
You may find this discussion helpful, when a similar issue was opened on github.

elasticsearch - add a constant mapping definition to a specific type in the mapping

I have a static mapping json that contains many entities.
for instance
{
"settings": {},
"mappings": {
"MyEntity": {
"properties": {
"date": {
"type": "date",
"format": "dateOptionalTime"
},
"name": {
"type": "string",
},
"tweet": {
"type": "string"
},
"user_id": {
"type": "long"
}
}
}
}
}
Where "MyEntity" is an example of one of many entities.
What I want is that every time an entity has the value:
"name": {
"type": "string",
},
this will be added:
"name": {
"type": "string",
"analyzer": "mm_name_analyzer",
"fields": {
"lc": {
"type": "string",
"analyzer": "case_insensitive_sort"
},
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
I don't want to add it to each entity field that is defined as string.
Is there a way to do it?
Here replace indexName with the index of your purpose or give an index name pattern
You can apply mapping to "__default_" which will make sure that the mapping is applied to all the types under the indeices the dynamic mapping is applied to.
curl -XPUT localhost:9200/_template/nameTemplate -d '{
"template": "indexName",
"mappings": {
"_default_": {
"dynamic_templates": [
{
"name_field": {
"match": "name",
"match_mapping_type": "string",
"mapping": {
"type": "string",
"analyzer": "mm_name_analyzer",
"fields": {
"lc": {
"type": "string",
"analyzer": "case_insensitive_sort"
},
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
]
}
}
}'