We have been trying to implement the encryption in Cosmos DB as per the below documentation
https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-always-encrypted?tabs=dotnet
We have been able to create the database with the encryption techniques and also the fields. Below is the code used
var databaseClient = cosmosClient.GetDatabase(createSqlIndexRequest.DatabaseName);
await databaseClient.CreateClientEncryptionKeyAsync(
keyName,
DataEncryptionKeyAlgorithm.AeadAes256CbcHmacSha256,
new EncryptionKeyWrapMetadata(
keyWrapProvider.ProviderName,
keyName,
keyVaultIdentifier));
var fieldEncryptionPathList = new List<ClientEncryptionIncludedPath>();
foreach (var fieldEncryptItem in createSqlIndexRequest.FieldNamesForEncryption)
{
fieldEncryptionPathList.Add(new ClientEncryptionIncludedPath()
{
Path = $"/{fieldEncryptItem}",
ClientEncryptionKeyId = keyName,
EncryptionType = EncryptionType.Deterministic.ToString(),
EncryptionAlgorithm = DataEncryptionKeyAlgorithm.AeadAes256CbcHmacSha256
});
}
ContainerProperties containerProperties = new ContainerProperties
{
PartitionKeyPath = createSqlIndexRequest.PartitionKey,
Id = createSqlIndexRequest.CollectionName,
ClientEncryptionPolicy = new ClientEncryptionPolicy(fieldEncryptionPathList),
IndexingPolicy = new Microsoft.Azure.Cosmos.IndexingPolicy()
{
IndexingMode = Microsoft.Azure.Cosmos.IndexingMode.None,
Automatic = false
}
};
var containerCheck = await databaseClient.CreateContainerIfNotExistsAsync(containerProperties, ruLevel.HasValue && !ruLevel.Value ? ruValue : null);
Once the above code runs, we can see the database and container created successfully.
We are trying to insert Documents using the DATA Explorer available in Azure Portal, a sample one
{
"id": "1234",
"job_id": 122,
"job_name": "mongo_cosmos_job",
"job_type": "onetimeDataOffload",
"source_connector_id": 98.0,
"source_connector_name": "mongo_src",
"source_type": "mongo",
"source_database": "company",
"destination_connector_id": 99.0,
"destination_connector_name": "cosmosmongo_serverless_dest",
"destination_type": "cosmos",
"destination_database": "mongo_cosmos_db",
"dag_run_id": "manual__2021-10-06T09:56:07.175777+00:00",
"task_id": "extract_and_load_company.test_collection",
"task_name": "extract_and_load_company.test_collection",
"table_name": "test_collection",
"stage": "load",
"record_count": 0,
"size_in_mb": 0.0,
"duration": 0.0,
"is_aggregated": false,
"is_test_dag": false,
"chunk_size": 5000.0,
"ru_level": "",
"ru": 0.0,
"job_execution_datetime": "2021-10-06 09:56:07.177",
"task_execution_datetime": "2021-10-06 09:56:07.177",
"job_status": "success",
"task_execution_endtime": "2021-10-06 09:56:42.650"
}
on click of Save, we get the following error
AzureCosmosDB Encryption Document Insert Error
The error message is as follows
the collection has ClientEncryptionPolicy set, but the document to be written isn't encrypted.
please help us in resolving the issue.
Related
I'm having a error when I use c# API clockify
When I create a Project with Memberships I get this error:
"{\"message\":\"Internal server error.\",\"code\":500}"
When i comment Memberships attribute the project it's created.
var assignees = new List<MembershipRequest>();
var member = new MembershipRequest();
member.MembershipStatus = MembershipStatus.Active;
member.MembershipType = "PROJECT";
member.UserId = "UserId";
assignees.Add(member);
var project = new ProjectRequest
{
Name = "ProjetName",
ClientId = "ClienteID",
IsPublic = false,
Billable = true,
Color = "#000000",
Memberships = assignees
};
var clockify = new ClockifyClient("APIKey");
var response = await clockify.CreateProjectAsync("WorkSpaceID", project).ConfigureAwait(true);
Memberships are a nested object.
So if it would work, the code would be
{
Name = "ProjetName",
ClientId = "ClienteID",
IsPublic = false,
Billable = true,
Color = "#000000",
"memberships": [{
"userId": assignee
}]
}
However this is not included in the endpoint, so what I did to make this work was:
PATCH /workspaces/{workspaceId}/projects/{projectId}/memberships
{
"memberships": [
{
"userId": assignee
}
]
}
I have the following table creation code for DynamoDB (C#):
client.CreateTable(new CreateTableRequest
{
TableName = tableName,
ProvisionedThroughput = new ProvisionedThroughput { ReadCapacityUnits = 20, WriteCapacityUnits = 10 },
KeySchema = new List<KeySchemaElement>
{
new KeySchemaElement
{
AttributeName = "RID",
KeyType = KeyType.HASH
}
}
,
AttributeDefinitions = new List<AttributeDefinition>
{
new AttributeDefinition {
AttributeName = "RID",
AttributeType = ScalarAttributeType.N
}
}
});
the data that gets populated into this table comes from this JSON:
[
{"RID": 208649, "CLI_RID": 935476, "PRT_DT": "VAL_AA", "DISTR": "INTERNAL"},
{"RID": 217427, "CLI_RID": 1009561, "PRT_DT": "VAL_BB", "DISTR": "INTERNAL", "STATE": "VAL_BD"},
{"RID": 223331, "CLI_RID": 1325477, "PRT_DT": "VAL_CB", "DISTR": "", "STATE": "VAL_CD", "FNAME": "VAL_CE", "START": "VAL_CF"},
{"RID": 227717, "CLI_RID": 1023478, "PRT_DT": "VAL_DB", "DISTR": "", "STATE": "VAL_DD"}
{"RID": 217462, "CLI_RID": 1009561, "PRT_DT": "VAL_BB", "DISTR": "", "STATE": "VAL_BD"},
{"RID": 218679, "CLI_RID": 1009561, "PRT_DT": "VAL_AA", "DISTR": "INTERNAL"},
{"RID": 222376, "CLI_RID": 1263978, "PRT_DT": "VAL_DB", "DISTR": "", "STATE": "VAL_DD"}
]
How would I Query or Filter for all records containing 1009561 in column "CLI_RID" and column "DISTR" <> "INTERNAL"?
There will be about 15 mil records in this DynamoDB table.
Is my table defined correctly for this query/filter?
Updated table creation:
// CLI_RIDIndex
var cli_ridIndex = new GlobalSecondaryIndex
{
IndexName = "cli_ridIndex",
ProvisionedThroughput = new ProvisionedThroughput
{
ReadCapacityUnits = 20,
WriteCapacityUnits = 10
},
KeySchema = {
new KeySchemaElement
{
AttributeName = "CLI_RID", KeyType = "HASH"
}
},
Projection = new Projection { ProjectionType = "ALL" }
};
client.CreateTable(new CreateTableRequest
{
TableName = tableName,
ProvisionedThroughput = new ProvisionedThroughput { ReadCapacityUnits = 20, WriteCapacityUnits = 10 },
KeySchema = new List<KeySchemaElement>
{
new KeySchemaElement
{
AttributeName = "RID",
KeyType = KeyType.HASH // Partiton Key (Unique)
},
new KeySchemaElement
{
AttributeName = "CLI_RID",
KeyType = KeyType.RANGE // Sort Key
}
}
,
AttributeDefinitions = new List<AttributeDefinition>
{
new AttributeDefinition {
AttributeName = "RID",
AttributeType = ScalarAttributeType.N
},
new AttributeDefinition {
AttributeName = "CLI_RID",
AttributeType = ScalarAttributeType.N
}
},
GlobalSecondaryIndexes = { cli_ridIndex }
});
But when attempting to query it,
var request = new QueryRequest
{
TableName = "TNAArchive",
KeyConditionExpression = "CLI_RID = :v_cli_rid",
ExpressionAttributeValues = new Dictionary<string, AttributeValue> {
{":v_cli_rid", new AttributeValue { S = "905466" }}}
};
var response = client.Query(request);
I get this error:
Query condition missed key schema element: RID
I guess I'm not really understanding how to do this.
According to your table structure, you won't be able to perform Query on the table but you have to Scan it which we need to avoid.
To perform Query you need to modify certain things
1) Add a Global Secondary Index(GSI) with the field CLI_RID as Hash
2) Now You Query GSI by passing CLI_RID and add query filter with condition <> your value
Here is the reference link.
Edit: Your Main table structure will be same no need to change, but you need to add one more GSI with Hash key as CLI_RID and project required table attribute.
Now you need to query your GSI instead of the table with a hash key as CLI_RID, you don't need to pass RID here.
here is the link on how to add GSI on table.
If CLI_RID is not present in a master table then that record will not be reflected in the GSI so no need to worry.
Edit 2: Just add (IndexName = NameOFYourIndex) attribute while querying and everything should work.
var request = new QueryRequest
{
TableName = "TNAArchive",
IndexName = "NameOfYourIndex",
KeyConditionExpression = "CLI_RID = :v_cli_rid",
ExpressionAttributeValues = new Dictionary<string, AttributeValue> {
{":v_cli_rid", new AttributeValue { S = "905466" }}}
};
Hope that helps
I am trying to publish forum replies to a specific thread, but I would like those reply documents to include extra information about the user that posted it.
I don't want to "save" that extra information on the reply itself but rather, publish an "improved" version of it.
I am doing something similar on client-side already with mycollection.find().map() and using the map function to embedded extra information on each of the returned documents, however, Meteor publish cannot seem to publish an array, only a Cursor, so the simple map function is off limits.
Is there a way to achieve this? Maybe a "map" function that returns a Cursor?
I am not using Meteor.methods so that I can have reactivity, because with them I could just return an array and use it as normal.
Here is an example of my code (that fails, but sends gives an idea of what I need):
Meteor.publish("forumthread", function(thread){
return forumReplies.find({thread: thread}).map(function(r){
// lets fill in additional data about each replies owner
var owner = Meteor.users.findOne({_id: r.owner});
if(!owner)
return; // no owner no reply..
if(!owner.forumStats){
owner.forumStats = {};
owner.forumStats.postCount = 0;
owner.forumStats.postLikes = 0;
owner.forumStats.title = "The Newbie";
owner.forumStats.tag = "Newbie";
Meteor.users.update({_id: owner._id}, {$set:{ forumStats:owner.forumStats }});
}
r.ownerid = owner._id;
r.ownerUsername = owner.username;
r.ownerPostCount = owner.forumStats.postCount;
r.ownerPostLikes = owner.forumStats.postLikes;
r.ownerTitle = owner.forumStats.title;
r.ownerTag = owner.forumStats.tag;
return r;
});
});
Thank you.
Ended up doing this (found out that Christian Fritz also suggested it):
Meteor.publish("serverforumthread", function(thread){
check(thread, String);
var replies = forumReplies.find({thread: thread});
var users = {};
replies.map(function(r){
users[r.owner] = r.owner;
});
var userids = _.map(users, function(value, key){ return value; });
var projectedFields = {_id:1, username:1, forumStats: 1, services: 0};
var usrs = Meteor.users.find({_id:{$in: userids}}, projectedFields);
var anyUpdateToUsers = false;
usrs.map(function(owner){
var changed = false;
if(!owner.username){
owner.username = owner.emails[0].address.split("#")[0];
changed = true;
}
//owner.forumStats = undefined;
if(!owner.forumStats){
owner.forumStats = {};
owner.forumStats.postCount = 0;
owner.forumStats.postLikes = 0;
owner.forumStats.title = "the newbie";
owner.forumStats.tag = "newbie";
owner.forumStats.img = "http://placehold.it/122x122";
changed = true;
}
if(changed){
anyUpdateToUsers = true;
Meteor.users.update({_id: owner._id}, {$set:{ forumStats:owner.forumStats }});
}
});
if(anyUpdateToUsers) // refresh it
usrs = Meteor.users.find({_id:{$in: userids}}, projectedFields);
usrs.map(function(owner){
console.log(owner);
});
return [replies, usrs];
});
It works great with the following client side:
Template.forumReplyOwner.helpers({
replyOwner: function(reply){
var owner = Meteor.users.findOne({_id: reply.owner});
console.log(reply, owner);
if(!owner || !owner.forumStats) return; // oh shait!
var r = {};
r.owner = owner._id;
r.ownerUsername = owner.username;
r.ownerPostCount = owner.forumStats.postCount;
r.ownerPostLikes = owner.forumStats.postLikes;
r.ownerTitle = owner.forumStats.title;
r.ownerTag = owner.forumStats.tag;
r.ownerImg = owner.forumStats.img;
return r;
},
ownerImgTab: function(){
return {src: this.ownerImg};
}
});
However, I am now facing another problem. Even tho I am restricting the fields I am publishing from the Users collection, it is still sending down the "services" field, that contains login data that shouldnt be sent, ideas?
I am publishing multi-user information (using Meteor.users collection) for the purpose of naming posts creators and have their names and other small details associated with those posts, but I do NOT want to publish the complete documents for each user as they have "secret" login information.
Here is the code I am using:
Meteor.publish("serverforumthread", function(thread){
check(thread, String);
var replies = forumReplies.find({thread: thread});
var users = {};
replies.map(function(r){
users[r.owner] = r.owner;
});
var userids = _.map(users, function(value, key){ return value; });
var projectedFields = {_id:1, username:1, forumStats: 1, services: 0};
var usrs = Meteor.users.find({_id:{$in: userids}}, projectedFields);
var anyUpdateToUsers = false;
usrs.map(function(owner){
var changed = false;
if(!owner.username){
owner.username = owner.emails[0].address.split("#")[0];
changed = true;
}
//owner.forumStats = undefined;
if(!owner.forumStats){
owner.forumStats = {};
owner.forumStats.postCount = 0;
owner.forumStats.postLikes = 0;
owner.forumStats.title = "the newbie";
owner.forumStats.tag = "newbie";
owner.forumStats.img = "http://placehold.it/122x122";
changed = true;
}
if(changed){
anyUpdateToUsers = true;
Meteor.users.update({_id: owner._id}, {$set:{ forumStats:owner.forumStats }});
}
});
if(anyUpdateToUsers) // refresh it
usrs = Meteor.users.find({_id:{$in: userids}}, projectedFields);
usrs.map(function(owner){
console.log(owner);
});
return [replies, usrs];
});
As you can see, I am only interested in publishing relies (posts) for a thread and their associated users username and small forumStats, I want to keep the "services" key secret, as it contains details that should not be published.
A sample output of the "console.log":
{ _id: 'hoRYFbRkXXbHYm8Ty',
createdAt: Tue Jun 03 2014 16:25:42 GMT+0100 (WEST),
emails: [ { address: 'somemail#gmail.com', verified: false } ],
forumStats:
{ postCount: 85,
postLikes: 5,
title: 'the newbie',
tag: 'newbie',
img: 'http://placehold.it/122x122' },
services:
{ password: { srp: [Object] },
resume: { loginTokens: [Object] } } }
What am I doing wrong?
Thank you.
Have a look at the examples in the field specifiers section of the docs, and give this a try:
var projectedFields = {fields: {username:1, forumStats: 1}};
You'll get _id for free, and it will only include the other fields that you specify. Note that you can't mix inclusion and exclusion options, meaning you can't have both 0's and 1's.
If that doesn't work, let me know and I'll look more carefully.
How can I get all child structure groups given a structure group TCM URI with Core Service?
I tried using this code:
ItemsFilterData sgFilter = new RepositoryItemsFilterData
{ ItemTypes = new[] { ItemType.StructureGroup },
Recursive = true,
BaseColumns = ListBaseColumns.Id };
XElement listXml;
using (CoreServiceClient client = _coreServiceProvider.GetCoreServiceClient())
{
listXml = XElement.Parse(
client.ProxyClient.GetListXml(structureGroupUri, sgFilter)
.OuterXml);
}
But I got an error saying "Unexpected item type: StructureGroup".
Starting from the Publication's URI, this works:
client.GetListXml("tcm:0-10-1", new RepositoryItemsFilterData {
ItemTypes = new[] { ItemType.StructureGroup },
Recursive = true,
BaseColumns = ListBaseColumns.Id
})
The trick is always to find the correct filter type, which in this case is RepositoryItemsFilterData.
Non-working sample
Starting from the Structure Group's URI, this returns the direct children Structure Groups. Note that that Recursive = true seems to be ignored here.
client.GetListXml("tcm:10-328-4", new OrganizationalItemItemsFilterData {
ItemTypes = new[] { ItemType.StructureGroup },
Recursive = true,
BaseColumns = ListBaseColumns.Id
})