Cosmos equivalent of map / select - azure-cosmosdb

clients is an array inside my doc, the following query
SELECT
f.id, f.clients
FROM f
where f.id ="35fb0733-dfa1-4932-9690-3ee5b05d89ff"
Returns
[
"id": "35fb0733-dfa1-4932-9690-3ee5b05d89ff",
{
"clients": [
{
"firstname": "Benjamin",
"surname": "Bob",
},
{
"firstname": "Rachael",
"surname": "Smith",
}
]
}
]
But I would like clients to look like :
"firstnames": [ "Benjamin", "Rachael" ]
"surnames": [ "Bob", "Smith" ]
Is this possible?

You could use the ARRAY expression w/ a subquery to achieve that.
Try this query:
SELECT
ARRAY(SELECT VALUE client.firstname FROM client IN f.clients) AS firstnames,
ARRAY(SELECT VALUE client.surname FROM client IN f.clients) AS surnames
FROM f

Just give an additional option, you could use stored procedure to get the results you want.
function sample() {
var collection = getContext().getCollection();
var isAccepted = collection.queryDocuments(
collection.getSelfLink(),
'SELECT f.id, c.firstname,c.surname FROM f join c in f.clients where f.id ="1"',
function (err, feed, options) {
if (err) throw err;
var map = {};
var firstname = [];
var surname =[];
if (!feed || !feed.length) {
var response = getContext().getResponse();
response.setBody('no docs found');
}
else {
for(var i=0;i<feed.length;i++){
map["id"] = feed[i].id;
firstname.push(feed[i].firstname);
surname.push(feed[i].surname);
}
map["firstname"] = firstname;
map["surname"] =surname;
var response = getContext().getResponse();
response.setBody(map);
}
});
if (!isAccepted) throw new Error('The query was not accepted by the server.');
}
Test Output:
{
"id": "1",
"firstname": [
"Benjamin",
"Rachael"
],
"surname": [
"Bob",
"Smith"
]
}

Related

aws glue job to import dynamodb data

We are trying to do DynamoDB migration from prod account to stage account.
In the source account, we are making use of "Export" feature of DDB to put the compressed .json.gz files into destination S3 bucket.
We have written a glue script which will read the exported .json.gz files and writes it to DDB table.
We are making the code generic, so we should be able to migrate any DDB table from prod to stage account.
As part of that process, while testing we are facing issues when we are trying to write a NUMBER SET data to target DDB table.
Following is the sample snippet which is raising ValidationException when trying to insert into DDB
from decimal import Decimal
def number_set(datavalue):
# datavalue will be ['0', '1']
set_of_values = set()
for value in datavalue:
set_of_values.add(Decimal(value))
return set_of_values
When running the code, we are getting following ValidationException
An error occurred while calling o82.pyWriteDynamicFrame. Supplied AttributeValue is empty, must contain exactly one of the supported datatypes (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ValidationException; Request ID: UKEU70T0BLIKN0K2OL4RU56TGVVV4KQNSO5AEMVJF66Q9ASUAAJG; Proxy: null)
However, if instead of Decimal(value) if we use int(value) then no ValidationException is being thrown and the job succeeds.
I feel that write_dynamic_frame_from_options will try to infer schema based on the values the element contains, if the element has "int" values then the datatype would be "NS", but if the element contains all "Decimal type" values, then it is not able to infer the datatype.
The glue job we have written is
dyf = glue_context.create_dynamic_frame_from_options(
connection_type="s3",
connection_options={
"paths": [file_path]
},
format="json",
transformation_ctx = "dyf",
recurse = True,
)
def number_set(datavalue):
list_of_values = []
for value in datavalue:
list_of_values.append(Decimal(value))
print("list of values ")
print(list_of_values)
return set(list_of_values)
def parse_list(datavalue):
list_of_values = []
for object in datavalue:
list_of_values.append(generic_conversion(object))
return list_of_values
def generic_conversion(value_dict):
for datatype,datavalue in value_dict.items():
if datatype == 'N':
value = Decimal(datavalue)
elif datatype == 'S':
value = datavalue
elif datatype == 'NS':
value = number_set(datavalue)
elif datatype == 'BOOL':
value = datavalue
elif datatype == 'M':
value = construct_map(datavalue)
elif datatype == 'B':
value = datavalue.encode('ascii')
elif datatype == 'L':
value = parse_list(datavalue)
return value
def construct_map(row_dict):
ddb_row = {}
for key,value_dict in row_dict.items():
# value is a dict with key as N or S
# if N then use Decimal type
ddb_row[key] = generic_conversion(value_dict)
return ddb_row
def map_function(rec):
row_dict = rec["Item"]
return construct_map(row_dict)
mapped_dyF = Map.apply(frame = dyf, f = map_function, transformation_ctx = "mapped_dyF")
datasink2 = glue_context.write_dynamic_frame_from_options(
frame=mapped_dyF,
connection_type="dynamodb",
connection_options={
"dynamodb.region": "us-east-1",
"dynamodb.output.tableName": destination_table,
"dynamodb.throughput.write.percent": "0.5"
},
transformation_ctx = "datasink2"
)
can anyone help us in how can we unblock from this situation?
Record that we are trying to insert
{
"region": {
"S": "to_delete"
},
"date": {
"N": "20210916"
},
"number_set": {
"NS": [
"0",
"1"
]
},
"test": {
"BOOL": false
},
"map": {
"M": {
"test": {
"S": "value"
},
"test2": {
"S": "value"
},
"nestedmap": {
"M": {
"key": {
"S": "value"
},
"nestedmap1": {
"M": {
"key1": {
"N": "0"
}
}
}
}
}
}
},
"binary": {
"B": "QUFBY2Q="
},
"list": {
"L": [
{
"S": "abc"
},
{
"S": "def"
},
{
"N": "123"
},
{
"M": {
"key2": {
"S": "value2"
},
"nestedmaplist": {
"M": {
"key3": {
"S": "value3"
}
}
}
}
}
]
}
}

How to count subdocuments recursively

I have documents in my db that look like this:
{
"id": "1"
"entityType": "node"
"childNodes": [
{
"id": "2"
"entityType": "node"
"childNodes": [
...
]
}
]
}
As a large tree structure.
I'd like to count the total number of documents and subdocuments in my collection that are of entityType = "Node".
My attempt is to get the data one level at a time manually:
SELECT VALUE COUNT(c.id) FROM c where CONTAINS(c.id, 'a|') and c.entityType = 'node'
SELECT VALUE COUNT(l.id) FROM c JOIN l in c.childNodes where CONTAINS(c.id, 'a|') and c.entityType = 'node'
SELECT VALUE COUNT(l2.id) FROM c JOIN l in c.childNodes JOIN l2 in l.childNodes where CONTAINS(c.id, 'a|') and c.entityType = 'node'
First of all, it's hard to find a smooth(direct) way to implement your needs.Surely,the manual way you mentioned in your question works.However,if you have too many layers of JSON nesting, or it's random,your way maybe inappropriate.
I would suggest you loop the result recursively to get the count of objects which contains "entityType": "node".For example, in the cosmos db stored procedure:
function sample(prefix) {
var collection = getContext().getCollection();
var isAccepted = collection.queryDocuments(
collection.getSelfLink(),
'SELECT c.childNodes FROM c where c.entityType = "node"',
function (err, feed, options) {
if (err) throw err;
if (!feed || !feed.length) {
var response = getContext().getResponse();
response.setBody('no docs found');
}
else {
var response = getContext().getResponse();
var count = {count:1};
loopChildNodes(feed,count);
response.setBody(count);
}
});
if (!isAccepted) throw new Error('The query was not accepted by the server.');
function loopChildNodes(array,count){
for (var i=0;i<array.length;i++){
console.log(count)
if(array[i].entityType == "node"){
count.count++;
}
if(array[i].childNodes!=null)
loopChildNodes(array[i].childNodes,count)
}
}
}
My test data:
Output:

How to alias array elements in CosmosDB documents?

I have the following document,
{
"VehicleDetailId": 1,
"VehicleDetail": [
{
"Id": 1,
"Make": "BMW"
},
{
"Id": 1,
"Model": "ABDS"
},
{
"Id": 1,
"Trim": "5.6L/ASMD"
},
{
"Id": 1,
"Year": 2008
}
]
}
I want to give aliases for the array elements, something like this,
{
"VehicleDetailId": 1,
"Type": "VehicleDetail",
"VehicleDetail": [
{
"MakeId": 1,
"MakeValue": "BMW"
},
{
"ModelId": 1,
"ModelValue": "ABDS"
},
{
"TrimId": 1,
"TrimValue": "5.6L/ASMD"
},
{
"YearId": 1,
"YearValue": 2008
}
]
}
The following query seems to work fine, but since Id is common for all, it is repeating every time.
SELECT c.vehicleDetailId, ARRAY(SELECT v.Id AS MakeId, v.Make AS MakeValue,
v.Id AS ModelId, v.Model AS ModelValue,
v.Id AS TrimId, v.Trim AS TrimValue,
v.Id AS YearId, v.Year AS YearValue
FROM v IN c.VehicleDetail) AS VehicleDetail
FROM c
How should I write the query so that the Id does not repeat every time, and I can fetch an element from a specific position?
You could use UDF to implement your needs.
Udf code:
function userDefinedFunction(array){
var returnArray = [];
for(var i=0;i<array.length;i++){
var obj = array[i];
var map = {};
if(obj.Make){
map["MakeId"]= obj.Id;
map["MakeValue"]= obj.Make;
}else if(obj.Model){
map["ModelId"]= obj.Id;
map["ModelValue"]= obj.Model;
}else if(obj.Trim){
map["TrimId"]= obj.Id;
map["TrimValue"]= obj.Trim;
}else if(obj.Year){
map["YearId"]= obj.Id;
map["YearValue"]= obj.Year;
}
returnArray.push(map);
}
return returnArray;
}
Sql:
SELECT c.VehicleDetailId,udf.test(c.VehicleDetail) AS VehicleDetail
FROM c
Output:

Looping through a list using foreach

I have a project that requires me to populate User information along with their personal information.
So far, I was able to loop through a list of users and personal information, but I was not able to populate null values.
For example:
public List<UserDetailModel> UserInformation()
{
List<UserDetailModel> userdetails = new List<UserDetailModel>();
var user = _context.User.Where(x => x.Id > 0).ToList()
foreach(var item in user)
{
var personaldetails = _context.PersonalDetails.Where(x => item.PId == x.PId).ToList();
foreach (var item2 in personaldetails)
{
UserDetailModel userModel = new UserDetailModel();
userModel.UserId = item.UserId;
userModel.Name = item.UserName;
userModel.PhoneNumber = item.Number;
userModel.CreditCardNumber = item2.CCNumber;
userModel.SIN = item2.SinNumber;
userdetails.Add(userModel);
}
}
return userdetails;
}
What I'm expecting is:
"userId": 1,
"name": "john"
"phoneNUmber": 123-123-1234,
"creditCardNumber": 44455544445554545,
"sin": 9589898568
"userId": 1,
"name": "john"
"phoneNUmber": ,
"creditCardNumber": 44455544445554545,
"sin": 9589898568
"userId": 1,
"name": "john"
"phoneNUmber": 123-123-1234,
"creditCardNumber": ,
"sin": 9589898568
"userId": 1,
"name": "john"
"phoneNUmber": 123-123-1234,
"creditCardNumber": 44455544445554545,
"sin":
But what I'm getting with the above code is:
"userId": 1,
"name": "john"
"phoneNUmber": 123-123-1234,
"creditCardNumber": 44455544445554545,
"sin": 9589898568
How can I get all users along with their null values?
I guess the logic was not implemented correctly, Two loops not required seems,
Public List<UserDetailModel> UserInformation(){
List<UserDetailModel> userdetails = new List<UserDetailModel>();
var user = (from user in _context.User
join personal in _context.PersonalDetails
on user.PId equals personal.PId
Where user.Id>0).ToList();
foreach(var item in user)
{
UserDetailModel userModel = new UserDetailModel();
userModel.UserId = item.UserId;
userModel.Name = item.UserName;
userModel.PhoneNumber = item.Number;
userModel.CreditCardNumber = item2.CCNumber;
userModel.SIN = item2.SinNumber;
userdetails.Add(userModel);
}
So when personal details null it won't enter in the loop.

How to insert into mongoDB from HTML page

var productDB = new Meteor.Collection('products'); //Want to insert into this DB
var ProductParameters = nodeDB.find({"ACTIVE" : 1, "VARIENTS.ACCESS" : "PUBLIC"}, { "VARIENTS.NAME": 1, _id : 0 } ); //Taking Paramters from Another DB
Template.dpVar.events = {
'click .addProduct' : function (e) {
e.preventDefault();
ProductParameters.forEach(function(){ **//This is my Question.How to insert into productDB the key values as {ProductParameters: Val of ProductParameters}**
console.log(ProductParameters);
var pvariable = {
pvariable: tmpl.find("#ProductParameters").value
};
productDB.insert(pvariable);
});
}
};
Problem:
I have created form from the Parameters of nodeDB.
I want to store the data from this new form in a new DB productDB.
I want to run a loop where all the ProductParameters are read from nodeDB and their corresponding values inserted in form by user are pushed into ProductDB as new Entry.
EDIT:
NodeDB has Templates:
db.nodes.insert([
{
"GEOLOCATION": {
"GEO_CODE": [],
"ACTIVE_GEOLOCATION": false
},
"META": {
"CATEGORY": "levis",
"DESCRIPTION": "dsad",
"PRIVACY": "PUBLIC",
"TEMPLATE_NAME": "B",
"TEMPLATE_GROUP": "Product",
"KEYWORDS": [
"sda"
],
"CREATEDBY": "",
"SUBCATEGORY": "Blue",
"PRODUCT_TEMPLATE_TYPE": "Consumable",
"UOM": "",
"TEMPLATE_SUBGROUP": ""
},
"VARIENTS": [
{
"COMMENT": "Demo",
"INDEX": 0,
"NAME": "Brand",
"IS_PARENT": false,
"DATATYPE": "Text",
"ACCESS": "PUBLIC",
"PARENT_VARIENT": "Parem",
"TYPE": "PERMANENT"
}
]
}
])
The form is generated only from the VARIENTS
The ProductDB would be {key,value} ={VARIENTS.NAME,value from UI}
There can be multiple VARIENTS as this contains only one "Brand"
instead of
var ProductParameters = nodeDB.find({"ACTIVE" : 1, "VARIENTS.ACCESS" : "PUBLIC"}, { "VARIENTS.NAME": 1, _id : 0 } );
add .fetch() at the end
var ProductParameters = nodeDB.find({"ACTIVE" : 1, "VARIENTS.ACCESS" : "PUBLIC"}, { "VARIENTS.NAME": 1, _id : 0 } ).fetch();

Resources