Increase data insert performance - azure-cosmosdb

We have created a code that loops in to CosmosDb records and inserts values to Sql db. But in the process, the data insertion perfomance is very slow. please help with suggestions, here is the code. This process is taking more than 4 hours for inserting a data of 4000 rows.
namespace PushNotificationUserInsert
{
class Program
{
static void Main(string[] args)
{
Task.Run(async () =>
{
#region Variables
var endpoint = ConfigurationManager.AppSettings["DocDbEndpoint"];
var masterKey = ConfigurationManager.AppSettings["DocDbMasterKey"];
var connetionString = ConfigurationManager.AppSettings["SQL_connetionString"].ToString();
// var useremail = "\"\"";
string path = #"xxxx.txt";
string[] useremails = File.ReadAllLines(path);
List<string> It = useremails.ToList();
var newIt = It.Distinct().ToList();
var channel = "\"msteam\"";
var cnn = new SqlConnection(connetionString);
var cnnInsert = new SqlConnection(connetionString);
string query = null;
cnn.Open();
foreach (var email in newIt)
{
query = "SELECT * FROM c where c.document.bot_js.mail = \"" + email + "\" AND CONTAINS(c.id," + channel + ", true)";
dynamic responses = "";
JSONModel.Rootobject records = null;
string conversationId = "";
//string email = "";
#endregion
#region Reading data from SQL
// cnn.Open();
SqlDataReader dataReader;
String Output = "";
SqlCommand command = new SqlCommand(#"SELECT [ConversatonReferenceJson] FROM [dbo].[ConversationReferences] where emailid like ''", cnn);
//command = new SqlCommand(sql, cnn);
dataReader = command.ExecuteReader();
while (dataReader.Read())
{
Output = dataReader.GetValue(0).ToString();
records = JsonConvert.DeserializeObject<JSONModel.Rootobject>(Output);
}
dataReader.Close();
dataReader.Dispose();
#endregion
string[] readText = File.ReadAllLines(path);
//List<string> It = readText.ToList();
//var newIt = It.Distinct().ToList();
//foreach (string email in newIt)
//{
Console.WriteLine(email);
try
{
#region reading data from Consmos DB
using (var client = new DocumentClient(new Uri(endpoint), masterKey))
{
responses = client.CreateDocumentQuery(UriFactory.CreateDocumentCollectionUri("Cosmosdbname", "cosmosdbtablebame"), query).ToList();
}
#endregion
#region Looping throught each cosmos DB records and insert value in SQL
foreach (var response in responses)
{
conversationId = response.id.Remove(0, 26);
records.conversation.id = conversationId;
cnnInsert.Open();
SqlCommand commInsert = new SqlCommand("INSERT INTO [ConversationReferences] VALUES " +
"(#ChannelId, #UserId, #ConverJson, #ID, #EmailID)", cnnInsert);
commInsert.Parameters.AddWithValue("#ChannelId", "xxxx");
commInsert.Parameters.AddWithValue("#UserId", "tobedeleted");
commInsert.Parameters.AddWithValue("#ConverJson", JsonConvert.SerializeObject(records));
commInsert.Parameters.AddWithValue("#ID", "tobedeleted");
commInsert.Parameters.AddWithValue("#EmailID", email);
commInsert.ExecuteNonQuery();
cnnInsert.Close();
Console.WriteLine("records updated for " + email);
}
#endregion
}
catch (Exception ex)
{
cnnInsert.Close();
throw;
}
finally
{ cnnInsert.Close(); }
Console.ReadKey();
}
}).Wait();
}
}

Related

Getting error "Database is locked" when I trye to save data i SQLITE DB

I have following class that connects to database but when I trye to call the method that saves data I get error "Database is locked". Could some one please help me to find the problem?
class Data
{
private SQLiteConnection con;
private void ConnectoDB()
{
String PathDB = Directory.GetCurrentDirectory();
PathDB += "\\SQLiteDB_MEDFit.db";
string cs = #"URI=file:" + PathDB;
string stm = "SELECT SQLITE_VERSION()";
con = new SQLiteConnection(cs);
con.Open();
var cmd = new SQLiteCommand(stm, con);
string version = cmd.ExecuteScalar().ToString();
}
public Boolean SaveToDatabase(string name, string number)
{
bool result = false;
try
{
ConnectoDB();
con.Execute("insert into People(name, number) values ('" + name+ "', '" + number+ "')");
con.Close();
result = true;
MessageBox.Show("Saved!");
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
return result;
}
}
The code that calls the "SaveToDatabase()"
string name = textBox1.Text();
string number = textBox2.Text();
Data connect = new Data();
connect.SavetoDB(name, number);

Web API to query SQL Server and return result is not working as expected

I am trying to connect to SQL Server from the Web API and return a result set as JSON. But my code shown here is not working as expected. I am trying to return the entire query response as a JSON:
[HttpGet]
public HttpResponseMessage Getdetails(string ROOM)
{
string commandText = "SELECT * from [TDB].[dbo].[results_vw] where ROOM = #ROOM_Data";
string connStr = ConfigurationManager.ConnectionStrings["TDBConnection"].ConnectionString;
var jsonResult = new StringBuilder();
using (SqlConnection connection = new SqlConnection(connStr))
{
SqlCommand command = new SqlCommand(commandText, connection);
command.Parameters.Add("#ROOM_Data", SqlDbType.VarChar);
command.Parameters["#ROOM_Data"].Value = ROOM;
connection.Open();
var reader = command.ExecuteReader();
if (!reader.HasRows)
{
jsonResult.Append("[]");
}
else
{
while (reader.Read())
{
jsonResult.Append(reader.GetValue(0).ToString());
}
}
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK);
response.Content = new StringContent(jsonResult.ToString());
connection.Close();
return response;
}
}
This code returns this result:
333838362692368203368203368203362692368203359544362692368203362692368203362692368203368203
Where I am expecting the JSON as
{"data":
[
{"R_ID":"368203","ROOM":"K2"},
{"R_ID":"368203","ROOM":"K2"}
]}
Now I created a model class called DatabaseResult to store the response but I am not sure how I can store the result in to the model class in the controller
public class DatabaseResult
{
public int r_id { get; set; }
public string room { get; set; }
}
The current result is because you are only return the the value from the first column of each row and adding it to the string builder.
Create a new instance of the model and populate it using the values from the reader for each row.
[HttpGet]
public IHttpActionResult Getdetails(string ROOM) {
string commandText = "SELECT * from [TDB].[dbo].[results_vw] where ROOM = #ROOM_Data";
string connStr = ConfigurationManager.ConnectionStrings["TDBConnection"].ConnectionString;
var jsonResult = new StringBuilder();
using (SqlConnection connection = new SqlConnection(connStr)) {
using (SqlCommand command = new SqlCommand(commandText, connection)) {
command.Parameters.Add("#ROOM_Data", SqlDbType.VarChar);
command.Parameters["#ROOM_Data"].Value = ROOM;
connection.Open();
List<DatabaseResult> records = new List<DatabaseResult>();
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new DatabaseResult {
r_id = (int)reader["r_id"],
room = (string)reader["room"],
//...other properties.
};
records.Add(row);
}
return Ok(records);
}
}
}
}
The above uses the column names as the indexer to get the values from the reader.

jQuery Bootgrid sorting, pagination and search functionality not working

I have a jQuery bootgrid implemented into my ASP.Net application which is filled using a Generic Handler.
I fill the bootgrid using the Generic Handler as follows:
$(function () {
var grid = $("#grid").bootgrid({
ajax: true,
ajaxSettings: {
method: "GET",
contentType: "application/json; charset=utf-8",
dataType: "json",
cache: false
},
url: "/MyHandler.ashx",
rowCount: [10, 50, 75, 100, 200, -1]
});
}
Here's MyHandler.ashx code:
public class RolesHandler : IHttpHandler
{
public void ProcessRequest(HttpContext context)
{
context.Response.ContentType = "text/json";
context.Response.Write(GetData());
}
public bool IsReusable
{
get
{
return false;
}
}
public string GetData()
{
var result = string.Empty;
var con = new SqlConnection();
var cmd = new SqlCommand();
var dt = new DataTable();
string sSQL = #"SELECT Id, Name
FROM dbo.AspNetRoles;";
try
{
using (var connection = THF.Models.SQLConnectionManager.GetConnection())
{
using (var command = new SqlCommand(sSQL, connection))
{
connection.Open();
command.CommandTimeout = 0;
var da = new SqlDataAdapter(command);
da.Fill(dt);
}
}
var sNumRows = dt.Rows.Count.ToString();
var sDT = JsonConvert.SerializeObject(dt);
result = "{ \"current\": 1, \"rowCount\": 10, \"rows\": " + sDT + ", \"total\": " + sNumRows + " }";
}
catch (Exception ex)
{
}
finally
{
cmd.Dispose();
THF.Models.SQLConnectionManager.CloseConn(con);
}
return result;
}
}
Basically all the important functionality of my bootgrid that worked before I implemented it the ajax way doesn't work anymore. Specifically the ordering, searching and pagination functionality aren't working at all without any errors.
As far as I know from a bit of research. This is because every time a search phrase is made, or a header is clicked (for ordering) etc. The bootgrid performs an ajax call.
Any idea on how to fix the functionality here?
After much work I ended up getting it working and this is the final code result:
public class RolesHandler : IHttpHandler
{
public void ProcessRequest(HttpContext context)
{
context.Response.ContentType = "text/json";
var current = context.Request.Params["current"];
var rowCount = context.Request.Params["rowCount"];
var orderById = context.Request.Params["sort[Id]"];
var orderByName = context.Request.Params["sort[Name]"];
var searchPhrase = context.Request.Params["searchPhrase"];
var orderBy = "Id";
var orderFrom = "ASC";
if (orderById != null)
{
orderBy = "Id";
orderFrom = orderById;
}
else if (orderByName != null)
{
orderBy = "Name";
orderFrom = orderByName;
}
context.Response.Write(GetData(current, rowCount, orderBy, orderFrom, searchPhrase));
}
public bool IsReusable
{
get
{
return false;
}
}
public string GetData(string current, string rowCount, string orderBy, string orderFrom, string searchPhrase)
{
var result = string.Empty;
var currentNum = Convert.ToInt32(current) - 1;
var temp = 0;
if (!"Id".Equals(orderBy, StringComparison.OrdinalIgnoreCase)
&& !"Name".Equals(orderBy, StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("orderBy is not a valid value");
if (!"desc".Equals(orderFrom, StringComparison.OrdinalIgnoreCase) && !"asc".Equals(orderFrom, StringComparison.OrdinalIgnoreCase))
throw new ArgumentException("orderFrom is not a valid value");
if (!int.TryParse(rowCount, out temp))
throw new ArgumentException("Rowcount is not a valid number");
var dt = new DataTable();
string sSQL = #"SELECT Id, Name
FROM dbo.AspNetRoles
WHERE Id LIKE #searchPhrase
OR Name LIKE #searchPhrase
ORDER BY " + orderBy + " " + orderFrom + #"
OFFSET ((" + currentNum.ToString() + ") * " + rowCount + #") ROWS
FETCH NEXT " + rowCount + " ROWS ONLY;";
using (var connection = THF.Models.SQLConnectionManager.GetConnection())
{
using (var command = new SqlCommand(sSQL, connection))
{
command.Parameters.Add(new SqlParameter("#searchPhrase", "%" + searchPhrase + "%"));
command.Parameters.Add(new SqlParameter("#orderBy", orderBy));
connection.Open();
command.CommandTimeout = 0;
var da = new SqlDataAdapter(command);
da.Fill(dt);
connection.Close();
}
}
var total = string.Empty;
string sSQLTotal = #"SELECT COUNT(*)
FROM dbo.Log
WHERE Id LIKE #searchPhrase
OR Name LIKE #searchPhrase;";
using (var connection = THF.Models.SQLConnectionManager.GetConnection())
{
using (var command = new SqlCommand(sSQLTotal, connection))
{
command.Parameters.Add(new SqlParameter("searchPhrase", "%" + searchPhrase + "%"));
connection.Open();
command.CommandTimeout = 0;
total = command.ExecuteScalar().ToString();
connection.Close();
}
}
var rows = JsonConvert.SerializeObject(dt);
return result = "{ \"current\": " + current + ", \"rowCount\": " + rowCount + ", \"rows\": " + rows + ", \"total\": " + total + " }";
}
}

session gets cleared once method finish

I have two methods in my webservice.cs page like below.
[WebMethod(EnableSession = true)]
public void checkEmployee(string emailId, string password)
{
List<Employee> ListEmp = new List<Employee>();
string cs = ConfigurationManager.ConnectionStrings["rushDB"].ConnectionString;
using (SqlConnection con = new SqlConnection(cs))
{
SqlCommand cmd = new SqlCommand("select * from Employees where Email='" + emailId + "' AND Password='" + password + "'", con);
string personName = null;
string personEmail = null;
string personDeptName = null;
con.Open();
SqlDataReader rdr = cmd.ExecuteReader();
if (rdr.HasRows == true)
{
while (rdr.Read())
{
Employee empChild = new Employee();
empChild.ID = Convert.ToInt16(rdr["ID"]);
empChild.Name = rdr["Name"].ToString();
empChild.Email = rdr["Email"].ToString();
empChild.Password = rdr["Password"].ToString();
empChild.DepartName = rdr["DepartName"].ToString();
personName = rdr["Name"].ToString();
personEmail = rdr["Email"].ToString();
personDeptName = rdr["DepartName"].ToString();
ListEmp.Add(empChild);
}
HttpContext.Current.Session["NamePerson"] = personName;
HttpContext.Current.Session["EmailPerson"] = personEmail;
HttpContext.Current.Session["DepartPerson"] = personDeptName;
JavaScriptSerializer js = new JavaScriptSerializer();
Context.Response.Write(js.Serialize(ListEmp));
}
}
}
[WebMethod(EnableSession = true)]
public void GetCurrentData()
{
Employee ListEmp = new Employee();
if (HttpContext.Current.Session["NamePerson"] != null)
ListEmp.Name = HttpContext.Current.Session["NamePerson"].ToString();
if (HttpContext.Current.Session["EmailPerson"] != null)
ListEmp.Email = HttpContext.Current.Session["EmailPerson"].ToString();
if (HttpContext.Current.Session["DepartPerson"] != null)
ListEmp.DepartName = HttpContext.Current.Session["DepartPerson"].ToString();
JavaScriptSerializer js = new JavaScriptSerializer();
Context.Response.Write(js.Serialize(ListEmp));
}
My issue is that the session created in checkEmployee gets vanished after that method checkEmployee finishes its execution.
I want to retrieve the session values in GetCurrentData but i am unable to do so. Any solution would help me. i have googled it but that didn't helped.
You need a cookie-jar to be able to retain session-state on Web Services. Depending on how you're consuming the service it can get quite complicated if you haven't done it before...
Have at look at the following articles:
http://www.codeproject.com/Articles/322436/RestSessionState
http://www.codeproject.com/Articles/188749/WCF-Sessions-Brief-Introduction

I need to fetch all users email of a particular group using AD in .net 2.0

I know there are lots of methods already given in stackoverflow but in my case all of them taking too long time. I post a method which takes less time but still it is too long to implement. Please help me so that it takes less execution time. Also take consideration that i am using .net 2.0 framework.
try
{
List<string> lstEmails = new List<string>();
string filter1 = string.Format("(anr={0})", "groupname");
DirectorySearcher searcher = new DirectorySearcher(entry);
searcher.Filter = filter1;
searcher.SearchScope = SearchScope.Subtree;
searcher.PropertiesToLoad.Add("mail");
IEnumerable res = (IEnumerable)searcher.FindOne().GetDirectoryEntry().Invoke("members");
//IEnumerable<string> rest = (IEnumerable<string>)res;
if (res != null)
{
try
{
int index = 0;
foreach (IEnumerable resl in res)
{
DateTime start = DateTime.Now;
DirectoryEntry dr = new DirectoryEntry(resl);
string strEmail = null;
if (dr.Properties["mail"].Value != null)
{
strEmail = dr.Properties["mail"].Value.ToString();
Console.WriteLine(strEmail);
DateTime stop = DateTime.Now;
Console.WriteLine((stop - start).TotalMinutes.ToString());
index++;
Console.WriteLine(index.ToString());
}
if (!string.IsNullOrEmpty(strEmail))
{
// groupMemebers.Add("sam",strEmail);
}
}
}
catch { }
}
}
catch { }
This is your suggested method Daro..
DirectoryContext context = new DirectoryContext(DirectoryContextType.Domain, domain, "domainname" + strLDAPUserName, strLDAPPassword);
DomainController controller = DomainController.FindOne(context);
DirectoryEntry entry = new DirectoryEntry(string.Format("LDAP://{0}",controller.Domain), strLDAPUserName, strLDAPPassword, AuthenticationTypes.Secure);
List<string> userList = new List<string>();
DateTime StartTime = DateTime.Now;
using (DirectorySearcher ds = new DirectorySearcher(entry))
{
ds.PropertiesToLoad.Add("mail");
ds.PageSize = 10000;
string DistingushiedName = "CN=" + groupName + ",OU=Users,dc=CompanyName,dc=com";
ds.Filter = "(&(objectClass=user)(memberof:1.2.840.113556.1.4.1941:="+DistingushiedName+"))";
ds.SearchScope = SearchScope.Subtree;
try
{
foreach (SearchResult user in ds.FindAll())
{
try
{
userList.Add(user.Path);//.Properties["mail"][0].ToString());
}
catch (Exception E)
{
throw new Exception(E.Message);
}
}
}
catch(Exception E)
{
throw new Exception(E.Message);
}
DateTime EndTime = DateTime.Now;
TimeSpan Dif = EndTime.Subtract(StartTime);
}
Here is your solution:-
string[] email = new string[0];
DirectoryEntry entry = new DirectoryEntry("LDAP://OU=Users,dc=me,dc=com", username, password);
string groupName = "GroupName";//Group NAme
DirectorySearcher groupSearch = new DirectorySearcher(entry);
groupSearch.Filter = "(SAMAccountName=" + groupName + ")";
groupSearch.PropertiesToLoad.Add("member");
SearchResult groupResult = groupSearch.FindOne(); // getting the members who belongs to the concern groupname
if (groupResult != null)
{
email = new string[groupResult.Properties["member"].Count]; //creatign an array to store all the email address
for (int iSearchLoop = 0; iSearchLoop < groupResult.Properties["member"].Count; iSearchLoop++)
{
string userName = groupResult.Properties["member"][iSearchLoop].ToString();
int index = userName.IndexOf(',');
userName = userName.Substring(0, index).Replace("CN=", "").ToString(); // the name of the user will be fetched.
DirectorySearcher search = new DirectorySearcher(entry);
search.Filter = "(name=" + userName + ")";
search.PropertiesToLoad.Add("mail");
SearchResult result = search.FindOne(); //finding the mail id
if (result != null)
{
email[iSearchLoop] = result.Properties["mail"][0].ToString(); //assigning the mail id to an array....
}
}
}
Hope this helps you
Easy enough (if your AD is 2003 R2 or higher):
List<string> userList = new List<string>();
DateTime StartTime = DateTime.Now;
using (DirectorySearcher ds = new DirectorySearcher(new DirectoryEntry ("GC://DC=YourDomain,DC=com")))
{
ds.PropertiesToLoad.Add("mail");
ds.PageSize = 10000;
ds.Filter = "(&(objectClass=user)(memberof:1.2.840.113556.1.4.1941:=YOUR_GROUP'S DN))";
ds.SearchScope = SearchScope.Subtree;
try
{
foreach (SearchResult user in ds.FindAll())
{
try
{
userList.Add(user.Path);//.Properties["mail"][0].ToString());
}
catch (Exception E)
{
throw new Exception(E.Message);
}
}
}
catch(Exception E)
{
throw new Exception(E.Message);
}
DateTime EndTime = DateTime.Now;
TimeSpan Dif = EndTime.Subtract(StartTime);
}
Replace YOUR_GROUP'S DN with the distiguished name of your group...
memberof:1.2.840.113556.1.4.1941:= is the "new" LDAP_MATCHING_RULE_IN_CHAIN operator, and retrieves all group members. Look here to see if your AD is ready and get more information.
Edit:
I gave you an answer, but an explanation might help further.
In general you should avoid ANR searches because they expand to large wildcard OR queries. Use them only if you have no idea which property contains the name you are searching for. They are much slower than explicit AND searches.
Secondly if you have more than one domain, you should turn off referral chasing unless you want to search through all domains until you get a hit. In this case it would be better to do a GC:// than a LDAP:// search to find the object you’re looking for, than do an LDAP search on that object. Depending on what you are looking for the GC query could well be enough
Edit 2:
Modified the code to give more error information and get the user path instead of E-Mail.
Hey this is the correct way...
try
{
List<string> ReturnArray = new List<string>();
DirectoryContext context = new DirectoryContext(DirectoryContextType.Domain, domainName, domainName + "\\" + UserName, Password);
DomainController controller = DomainController.FindOne(context);
string LDAPAddress = string.Format("LDAP://{0}", controller.Domain);
DirectoryEntry deDirEntry = new DirectoryEntry(LDAPAddress, UserName, Password);
deDirEntry.AuthenticationType = AuthenticationTypes.Secure;
DirectorySearcher mySearcher = new DirectorySearcher(deDirEntry);
mySearcher.PropertiesToLoad.Add("distinguishedName");
string sFilter = String.Format("(&(objectcategory=group)(cn=" + GroupName + "))");
mySearcher.Filter = sFilter;
mySearcher.Sort.Direction = SortDirection.Ascending;
mySearcher.Sort.PropertyName = "cn";
SearchResult result;
DirectoryEntry ResultEntry;
result = mySearcher.FindOne();
ResultEntry = result.GetDirectoryEntry();
GroupName = ResultEntry.Properties["distinguishedName"].Value.ToString();
mySearcher = new DirectorySearcher(deDirEntry);
mySearcher.PropertiesToLoad.Add("cn");
mySearcher.PropertiesToLoad.Add("mail");
sFilter = String.Format("(&(objectClass=person)(memberOf={0}))", GroupName);
mySearcher.Filter = sFilter;
mySearcher.Sort.Direction = SortDirection.Ascending;
mySearcher.Sort.PropertyName = "cn";
SearchResultCollection results;
results = mySearcher.FindAll();
foreach (SearchResult resEnt in results)
{
ResultPropertyCollection propcoll = resEnt.Properties;
foreach (string key in propcoll.PropertyNames)
{
if (key == "mail")
{
foreach (object values in propcoll[key])
{
if (!String.IsNullOrEmpty(values.ToString()))
{
ReturnArray.Add(values.ToString());
Console.WriteLine(values.ToString());
}
}
}
}
}
return ReturnArray;
}
catch
{
return null;
}
Thanks everyone for your valuable suggesstion

Resources