I'm very newbie to ElasticSearch. This is how my data looks like.how can I map and query on the following data.
[parameters] => Array
(
[0] => Array
(
[param_id] => "Browser"
[param_values] => Array
(
[0] => "Firefox"
)
)
[1] => Array
(
[param_id] => "BrowserVersion"
[param_values] => Array
(
[0] => "39"
)
)
[2] => Array
(
[param_id] => "OS"
[param_values] => Array
(
[0] => "Windows"
)
)
[3] => Array
(
[param_id] => "Softwares"
[param_values] => Array
(
[0] => "Java"
[1] => "Oracle"
[2] => "PHP"
)
)
)
I have to get the results of the following query.
The Users which are using "OS" as "Windows" and "Softwares" as "Java". How to map these data and query?
Below is the Mapping JSON:
{
"response": {
"properties": {
"profile_id" : {"type" : "long"},
"timestamp" : {"type" : "string"},
"parameters" : {
"type": "nested",
"properties":{
"param_id":{"type": "string"},
"param_values": {
"type": "string"
}
}
}
}
}
}
Below is my Query JSON:
{ "query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{
"term": {
"parameters.param_id": "Softwares"
}
},
{
"term": {
"parameters.param_values": "Java"
}
}
]
}
}
}
}
]
}
}
} }}
So let's start off by creating the index using
curl -XPUT localhost:9200/responses -d '{
"mappings": {
"response": {
"properties": {
"profile_id": {
"type": "long"
},
"timestamp": {
"type": "string"
},
"parameters": {
"type": "nested",
"properties": {
"param_id": {
"type": "string"
},
"param_values": {
"type": "string"
}
}
}
}
}
}
}'
And then we index your sample document:
curl -XPUT localhost:9200/responses/response/1 -d '{
"profile_id": 1,
"timestamp": "2015-01-01T00:00:00",
"parameters": [
{
"param_id": "Browser",
"param_values": [
"Firefox"
]
},
{
"param_id": "BrowserVersion",
"param_values": [
"39"
]
},
{
"param_id": "OS",
"param_values": [
"Windows"
]
},
{
"param_id": "Softwares",
"param_values": [
"Java",
"Oracle",
"PHP"
]
}
]
}'
You've started off very well and your query is almost correct. If you want to query for users who are using "Windows" as "OS" and "Java" as "Softwares", you already have the constraint on the latter, now you just need one more contraint for the former (i.e. OS=Windows)
{
"size": 30,
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{
"term": {
"parameters.param_id": "os"
}
},
{
"term": {
"parameters.param_values": "windows"
}
}
]
}
}
}
},
{
"nested": {
"path": "parameters",
"query": {
"bool": {
"must": [
{
"term": {
"parameters.param_id": "softwares"
}
},
{
"term": {
"parameters.param_values": "java"
}
}
]
}
}
}
}
]
}
}
}
}
}
Related
{
"id":{"N": "1"},
"attributes": {
"L": [
{
"M": {
"name": { "S": "AA" }
}
},
{
"M": {
"name": { "S": "BB" }
}
}
]
}
},
{
"id":{"N": "1"},
"attributes": {
"L": [
{
"M": {
"name": { "S": "BB" }
}
}
]
}
}
With the above data in the same partition( id as the partition key), How can I find records where any of attributes has name = 'BB' ?
I can filter by Nth item, for example.
KeyConditionExpression: 'id = :myId',
ExpressionAttributeValues: {'myId':{N:'1'},'myValue':{S:'BB'}},
ExpressionAttributeNames:{'#name': 'name'},
FilterExpression: 'attributes[0].#name=:myValue'
This would return only 2nd item. But is there a way to filter by ANY item in the array? It should return both records.
Tried set FilterExpression to attributes[*].#name=:myValue or attributes.#name=:myValue, neither works.
When I query a resolver in my GraphQL API, in which I have added a $util.error($ctx) to return the context object, I get the following result (removed unnecessary values).
{
"data": {
"listXData": null
},
"errors": [
{
"message": {
"arguments": {},
"args": {},
"info": {
"fieldName": "listXData",
"variables": {},
"parentTypeName": "Query",
"selectionSetList": [
"items",
"items/id",
"items/createdAt",
"items/updatedAt",
"nextToken"
],
"selectionSetGraphQL": "{\n items {\n id\n createdAt\n updatedAt\n }\n nextToken\n}"
},
"request": {...},
"identity": {
"sub": "",
"issuer": "",
"username": "013fe9d2-95f7-4885-83ec-b7e2e0a1423f",
"sourceIp": "",
"claims": {
"origin_jti": "",
"sub": "",
"event_id": "",
"token_use": "",
"scope": "",
"auth_time": ,
"iss": "",
"exp": ,
"iat": ,
"jti": "",
"client_id": "",
"username": "013fe9d2-95f7-4885-83ec-b7e2e0a1423f"
},
"defaultAuthStrategy": "ALLOW"
},
"stash": {},
"source": null,
"result": {
"items": [],
"scannedCount": 0,
"nextToken": null
},
"error": null,
"prev": {
"result": {}
}
},
"errorType": null,
"data": null,
"errorInfo": null,
"path": [
"listXData"
],
"locations": [
{
"line": 2,
"column": 3,
"sourceName": "GraphQL request"
}
]
}
]
}
As you can see, the username is an ID, however I would prefer to (also) have the email. Is it possible to get the user email (within the Velocity template)?
Let me know if I need to add more details or if my question is unclear.
The identity context only returns back the Cognito username for the user pool. You will need to setup pipeline functions to perform additional queries to get your user information. Here is one intro to setting them up.
At this point, it seems that it is not possible to do this purely by vtl.
I have implemented it using a lambda function, as follow:
Lambda function (node):
/* Amplify Params - DO NOT EDIT
ENV
REGION
Amplify Params - DO NOT EDIT */
const aws = require('aws-sdk')
const cognitoidentityserviceprovider = new aws.CognitoIdentityServiceProvider({
apiVersion: '2016-04-18',
region: 'eu-west-1'
})
exports.handler = async (context, event, callback) => {
if (!context.identity?.username) {
callback('Not signed in')
}
const params = {
'AccessToken': context.request.headers.authorization
}
const result = await cognitoidentityserviceprovider.getUser(params).promise()
const email = result.UserAttributes.find(attribute => attribute.Name === 'email')
callback(null, JSON.stringify({ email }))
}
CustomResources.json
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "An auto-generated nested stack.",
"Metadata": {...},
"Parameters": {...},
"Resources": {
"GetEmailLambdaDataSourceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"RoleName": {
"Fn::If": [
"HasEnvironmentParameter",
{
"Fn::Join": [
"-",
[
"GetEmail17ec",
{
"Ref": "GetAttGraphQLAPIApiId"
},
{
"Ref": "env"
}
]
]
},
{
"Fn::Join": [
"-",
[
"GetEmail17ec",
{
"Ref": "GetAttGraphQLAPIApiId"
}
]
]
}
]
},
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "appsync.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
},
"Policies": [
{
"PolicyName": "InvokeLambdaFunction",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": {
"Fn::If": [
"HasEnvironmentParameter",
{
"Fn::Sub": [
"arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:GetEmail-${env}",
{
"env": {
"Ref": "env"
}
}
]
},
{
"Fn::Sub": [
"arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:GetEmail",
{}
]
}
]
}
}
]
}
}
]
}
},
"GetEmailLambdaDataSource": {
"Type": "AWS::AppSync::DataSource",
"Properties": {
"ApiId": {
"Ref": "AppSyncApiId"
},
"Name": "GetEmailLambdaDataSource",
"Type": "AWS_LAMBDA",
"ServiceRoleArn": {
"Fn::GetAtt": [
"GetEmailLambdaDataSourceRole",
"Arn"
]
},
"LambdaConfig": {
"LambdaFunctionArn": {
"Fn::If": [
"HasEnvironmentParameter",
{
"Fn::Sub": [
"arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:GetEmail-${env}",
{
"env": {
"Ref": "env"
}
}
]
},
{
"Fn::Sub": [
"arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:GetEmail",
{}
]
}
]
}
}
},
"DependsOn": "GetEmailLambdaDataSourceRole"
},
"InvokeGetEmailLambdaDataSource": {
"Type": "AWS::AppSync::FunctionConfiguration",
"Properties": {
"ApiId": {
"Ref": "AppSyncApiId"
},
"Name": "InvokeGetEmailLambdaDataSource",
"DataSourceName": "GetEmailLambdaDataSource",
"FunctionVersion": "2018-05-29",
"RequestMappingTemplateS3Location": {
"Fn::Sub": [
"s3://${S3DeploymentBucket}/${S3DeploymentRootKey}/pipelineFunctions/${ResolverFileName}",
{
"S3DeploymentBucket": {
"Ref": "S3DeploymentBucket"
},
"S3DeploymentRootKey": {
"Ref": "S3DeploymentRootKey"
},
"ResolverFileName": {
"Fn::Join": [
".",
[
"InvokeGetEmailLambdaDataSource",
"req",
"vtl"
]
]
}
}
]
},
"ResponseMappingTemplateS3Location": {
"Fn::Sub": [
"s3://${S3DeploymentBucket}/${S3DeploymentRootKey}/pipelineFunctions/${ResolverFileName}",
{
"S3DeploymentBucket": {
"Ref": "S3DeploymentBucket"
},
"S3DeploymentRootKey": {
"Ref": "S3DeploymentRootKey"
},
"ResolverFileName": {
"Fn::Join": [
".",
[
"InvokeGetEmailLambdaDataSource",
"res",
"vtl"
]
]
}
}
]
}
},
"DependsOn": "GetEmailLambdaDataSource"
},
"IsOrganizationMember": {
"Type": "AWS::AppSync::FunctionConfiguration",
"Properties": {
"FunctionVersion": "2018-05-29",
"ApiId": {
"Ref": "AppSyncApiId"
},
"Name": "IsOrganizationMember",
"DataSourceName": "PermissionsPerOrganizationTable",
"RequestMappingTemplateS3Location": {
"Fn::Sub": [
"s3://${S3DeploymentBucket}/${S3DeploymentRootKey}/resolvers/Query.isOrganizationMember.req.vtl",
{
"S3DeploymentBucket": {
"Ref": "S3DeploymentBucket"
},
"S3DeploymentRootKey": {
"Ref": "S3DeploymentRootKey"
}
}
]
},
"ResponseMappingTemplateS3Location": {
"Fn::Sub": [
"s3://${S3DeploymentBucket}/${S3DeploymentRootKey}/resolvers/Query.isOrganizationMember.res.vtl",
{
"S3DeploymentBucket": {
"Ref": "S3DeploymentBucket"
},
"S3DeploymentRootKey": {
"Ref": "S3DeploymentRootKey"
}
}
]
}
}
},
"OrganizationAccessPipeline": {
"Type": "AWS::AppSync::Resolver",
"Properties": {
"ApiId": {
"Ref": "AppSyncApiId"
},
"TypeName": "Query",
"Kind": "PIPELINE",
"FieldName": "listXData",
"PipelineConfig": {
"Functions": [
{
"Fn::GetAtt": [
"InvokeGetEmailLambdaDataSource",
"FunctionId"
]
},
{
"Fn::GetAtt": [
"IsOrganizationMember",
"FunctionId"
]
}
]
},
"RequestMappingTemplate": "{}",
"ResponseMappingTemplate": "$util.toJson($ctx.result)"
}
}
},
"Conditions": {...},
"Outputs": {...}
}
The lambda is created with the CLI and IsOrganizationMember is a regular VTL which has the user email in the $context.prev.result.
I have problem with my queries when I'm using " or ' - then I expect match_phrase, but I don't know how I can retrieve posts when I'm using match_phrase with &
For example I'm using Something & Something as phrase, and when I'm didn't using ' and " I can see posts with Something & Something but there I'm using multi_match.
Something what I've tried:
{
"from": 0,
"size": 10,
"sort": {
"post_date": {
"order": "desc"
}
},
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{
"match_phrase": {
"query": "Something & Something"
}
}
]
}
},
"exp": {
"post_date_gmt": {
"scale": "270d",
"decay": 0.5,
"offset": "90d"
}
},
"score_mode": "avg",
"boost_mode": "sum"
}
},
"post_filter": {
"bool": {
"must": [
{
"terms": {
"post_type.raw": [
"post"
]
}
},
{
"terms": {
"post_status": [
"publish"
]
}
}
]
}
}
}
But this doesn't return any post, and returning hits total 0. Anyone have any idea, or suggestions, what I'm doing wrong ?
match_phrase is very restrictive and in most of cases is recommended to use it inside a should clause to increase the score instead of a must, because it requires the user to type the value exactly as it is.
Example document
POST test_jakub/_doc
{
"query": "Something & Something",
"post_type": {
"raw": "post"
},
"post_status": "publish",
"post_date_gmt": "2021-01-01T12:10:30Z",
"post_date": "2021-01-01T12:10:30Z"
}
With this document searching for "anotherthing Something & Something" will return no results, that's why is a bad idea to use match_phrase here.
You can take 2 approaches
If you need this kind of tight queries take a look to the slop parameter that adds some flexibility to the match_phrase query allowing omit or transpose words in the phrase
Switch to a regular match query (recommended). In most cases this will work fine, but if you want to do extra score to the phrase matches you can add it as a should clause.
POST test_jakub/_search
{
"from": 0,
"size": 10,
"sort": {
"post_date": {
"order": "desc"
}
},
"query": {
"function_score": {
"query": {
"bool": {
"should": [
{
"match_phrase": {
"query": {
"query": "anotherthing something & something",
"slop": 2
}
}
}
],
"must": [
{
"match": {
"query": "anotherthing something & something"
}
}
]
}
},
"exp": {
"post_date_gmt": {
"scale": "270d",
"decay": 0.5,
"offset": "90d"
}
},
"score_mode": "avg",
"boost_mode": "sum"
}
},
"post_filter": {
"bool": {
"must": [
{
"terms": {
"post_type.raw": [
"post"
]
}
},
{
"terms": {
"post_status": [
"publish"
]
}
}
]
}
}
}
Last advice is to avoid using "query" as field name because leads to confusion and will break Kibana autocomplete on Dev Tools.
I'm trying to filter out a query based on a nested object (no array). I'm currently using AppSync and DynamoDB and the expression with expression values are executed correctly. But the filtering doesn't seem to work.
This is the sample data I'm trying to get (Filter by indicator.id):
Here's my query:
{
"version": "2017-02-28",
"operation": "Query",
"query": {
"expression": "pk = :pk and begins_with(sk, :sk)",
"expressionValues": {
":pk": { "S": "tenant:5fc30406-346c-42e2-8083-fda33ab6000a" },
":sk": {
"S": "school-year:2019-2020:grades:bVgA9abd:subject:m_kpc1Ae6:indicator:"
}
}
},
"filter": {
"expression": " contains(#indicatorId, :sk1) or contains(#indicatorId, :sk2) or contains(#indicatorId, :sk3)",
"expressionNames": { "#indicatorId": "indicator" },
"expressionValues": {
":sk1": {
"M": { "id": { "S": "07c658dd-999f-4e6f-95b8-c6bae422760a" } }
},
":sk2": {
"M": { "id": { "S": "0cf9f670-e284-4a93-b297-5e4a40c50228" } }
},
":sk3": { "M": { "id": { "S": "cd7902be-6512-4b47-b29d-40aff30c73e6" } } }
}
}
}
I've also tried:
{
"version": "2017-02-28",
"operation": "Query",
"query": {
"expression": "pk = :pk and begins_with(sk, :sk)",
"expressionValues": {
":pk": { "S": "tenant:5fc30406-346c-42e2-8083-fda33ab6000a" },
":sk": {
"S": "school-year:2019-2020:grades:bVgA9abd:subject:m_kpc1Ae6:indicator:"
}
}
},
"filter": {
"expression": " contains(#indicatorId, :sk1) or contains(#indicatorId, :sk2) or contains(#indicatorId, :sk3)",
"expressionNames": { "#indicatorId": "indicator.id" },
"expressionValues": {
":sk1": { "S": "07c658dd-999f-4e6f-95b8-c6bae422760a" },
":sk2": { "S": "0cf9f670-e284-4a93-b297-5e4a40c50228" },
":sk3": { "S": "cd7902be-6512-4b47-b29d-40aff30c73e6" }
}
}
}
I've also tried searching around StackOverflow, and Amazon forums and haven't found it directly to my problem:
How to filter by elements in an array (or nested object) in DynamoDB
Nested Query in DynamoDB returns nothing
Referring to this answer.enter link description here
According to DDB Nested Attributes doc, the filter expression should look like the following format:
"filter" : {
"expression" : "#path.#filter = :${fp}", ## filter path parent.target = :target
"expressionNames": {
"#path" : "${path}",
"#filter" : "${fp}"
},
"expressionValues" : {
":${fp}" : $util.dynamodb.toDynamoDBJson(${$target[$fp].eq}) ## :target : value to filter for
}
}
I have a problem with dynamic date tampletes
I'm using ElasticSearch 6.2.4
My steps:
1) Create index with next settings:
PUT /test1
{
"settings": {
"index":{
"number_of_shards" : 9,
"number_of_replicas" : 0,
"max_rescore_window" : 2000000000,
"max_result_window" : 2000000000
}
},
"mappings": {
"files": {
"properties": {
"Дата добавления в БД": {
"type": "date"
}
},
"numeric_detection": true,
"dynamic_templates": [
{
"integers": {
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"dates": {
"match_mapping_type": "date",
"mapping": {
"format": "yyyy-MM-dd HH:mm:ss||yyyy/MM/dd HH:mm:ss||yyyyMMdd_HH:mm:ss",
"type": "date"
}
}
}
]
}
}
}
2) Try to put new records (I have only one)
POST /test1/files/_bulk
{"create":{"_index":"test1","_type":"files","_id":"0"}}
{"Дата добавления в БД":"2019/04/12 11:42:21"}
3) So, I have next output:
{
"took": 1,
"errors": true,
"items": [
{
"create": {
"_index": "test1",
"_type": "files",
"_id": "0",
"status": 400,
"error": {
"type": "mapper_parsing_exception",
"reason": "failed to parse [Дата добавления в БД]",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Invalid format: \"2019/04/12 11:42:21\" is malformed at \"/04/12 11:42:21\""
}
}
}
}
]
}
I can't understand where is my mistake??
I tried to find some information about this problem in Google, unfortunately, I have no solves of this problem. Maybe, this question is so stupid, but I've already broken my brain.
Please, help me...
I can't fully understand, but this option work:
{
"settings": {
"index":{
"number_of_shards" : 9,
"number_of_replicas" : 0,
"max_rescore_window" : 2000000000,
"max_result_window" : 2000000000
}
},
"mappings": {
"files": {
"dynamic_date_formats": ["yyyy-MM-dd HH:mm:ss","yyyy/MM/dd HH:mm:ss", "yyyyMMdd_HH:mm:ss"],
"numeric_detection": true,
"date_detection": true,
"dynamic_templates": [
{
"integers": {
"match_mapping_type": "long",
"mapping": {
"type": "long"
}
}
},
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
}
}
}
Link to documentation:
https://www.elastic.co/guide/en/elasticsearch/reference/6.2/dynamic-field-mapping.html
Thanks for attention :)