How to download and extract large files from email using window service - asp.net

Question

I am using the following code in window service to download zip file with particular format from email attachments:
Method to get Emails:
public void GetdownloadfromGmail()
{
try
{
ConfigDetails ConfigDetails = new ConfigDetails();
DataTable SecTable = ConfigDetails.GetConfigTableCol(Constant.Seccurity);
using (IDbConnection con = Connections.Getsynchconnection())
{
try
{
con.Open();
ConnectionState state = con.State;
con.Close();
if (state == ConnectionState.Open)
{
SentmailCount = 0;
}
}
catch (Exception ex)
{
Common.Synch.Factory.LogHelpFactory.log.Error(ex.Message.ToString());
if (SentmailCount == 0)
{
string ContentMessage = "SqlConnection is Not Open :" + ex.Message.ToString();
Process SharedProc = new Process();
string EmailID = SharedProc.GetReceipientmailid(new Context(new MailContext()), "", 3);
string[] Receipients = EmailID.Split(',');
//time being profile,pwd is hard coded
GmailClient.GmailClient gc = new GmailClient.GmailClient();
/*Sending attachment(s) with the mail*/
gc.SMTPHost = SecTable.Rows[0]["SmtpHost"].ToString();
gc.SMTPPortNumber = Convert.ToInt32(SecTable.Rows[0]["SmtpPort"].ToString());
bool IsEnableSsl = Convert.ToBoolean(SecTable.Rows[0]["IsEnableSsl"].ToString());
System.Net.NetworkCredential credentials = new System.Net.NetworkCredential(SecTable.Rows[0]["Username"].ToString(), SecTable.Rows[0]["Password"].ToString());
System.Net.Mail.MailAddress address = new System.Net.Mail.MailAddress(SecTable.Rows[0]["Username"].ToString(), SecTable.Rows[0]["DisplayName"].ToString());
gc.sendMail("Connection", Receipients, ContentMessage, null, "", credentials, address, true, IsEnableSsl);
SentmailCount = 1;
}
return;
}
}
Common.Synch.Factory.LogHelpFactory.log.Info("Start Download Mails From Mailbox");
//ConfigDetails ConfigDetails = new ConfigDetails();
//time being profile,pwd is hard coded
// DataTable SecTable = ConfigDetails.GetConfigTableCol(Constant.Seccurity);
if (SecTable.Columns["UserName"] == null && SecTable.Columns["Password"] == null)
{
throw (new Exception("Invalid Profile settings"));
}
using (GmailClient.GmailClient gc = new GmailClient.GmailClient())
{
gc.UseSSL = true;
/*Part to fetch mails from gmail; List<MailMessage> will have all the mails fetched which can be used later to send mails.
The attachments get downloaded as the mails are read.
The MailMessage only includes the basic header information along with the number of attachments processed.
Once all emails are fetched, the client disconnects automatically*/
//Following property will not delete the email from gmail server if set to False, but deletes if it set to True
gc.SMTPHost = SecTable.Rows[0]["IncomingEmailHost"].ToString();
gc.SMTPPortNumber = Convert.ToInt32(SecTable.Rows[0]["IncomingEmailPort"].ToString());
gc.DeleteProcessedMails = Convert.ToBoolean(SecTable.Rows[0]["DeleteEmail"].ToString());
ConfigDetails = new ConfigDetails();
TempPath = ConfigDetails.GetPath(Constant.TempFolder);
//Default AttachmentPath is the path of dll directory. Howeve it can be set exclusilvely as below.
gc.AttachmentPath = TempPath + "\\";
string strUname = SecTable.Rows[0]["Username"].ToString();
string strPwd = SecTable.Rows[0]["Password"].ToString();
gc.Connect(strUname, strPwd);
gc.CopyProcessedMails = Convert.ToBoolean(SecTable.Rows[0]["CopyProcessedEmail"].ToString()); ;
gc.CopyMailTargetFolder = Convert.ToString(SecTable.Rows[0]["CopyMailTargetFolder"].ToString()); ;
Common.Synch.Factory.LogHelpFactory.log.Info("Test ");
if (gc.IsConnected)
{
Common.Synch.Factory.LogHelpFactory.log.Info("Connected");
List<MailMessage> mails = gc.GetMailList();
Common.Synch.Factory.LogHelpFactory.log.Info("Unread Mail Count:" + mails.Count.ToString()); try
{
foreach (MailMessage Mail in mails)
{
try
{
isDeleted = false;
if (null != Mail.Subject)
Common.Synch.Factory.LogHelpFactory.log.Info(Mail.Subject.ToString());
else
Common.Synch.Factory.LogHelpFactory.log.Info(Mail.FromEmail + " (no subject)");
GetMaiilboxtotable(Mail);
//if (isDeleted)
//{
//Mail.UnRead = false;
//Mail.Delete();
//}
}
catch (Exception Ex)
{
//if (Ex.Message == "Could not save attachment to a file")
//{
Common.Synch.Factory.LogHelpFactory.log.Info("Error while processing email:" + Ex.Message.ToString());
Common.Synch.Factory.LogHelpFactory.log.Info("Subject:" + Mail.Subject + ", From:" + Mail.FromName);
//Mail.UnRead = false;
//Mail.Delete();
//}
}
}
}
//if (ListofMails.Count > 0)
//{
//OutlookClient.MovFolderselection();
//}
catch (Exception Ex)
{
//if (ListofMails != null)
//{
// Marshal.ReleaseComObject(ListofMails);
//}
//ListofMails = null;
////OutlookClient.Dispose();
mails = null;
gc.Dispose();
Common.Synch.Factory.LogHelpFactory.log.Error("SearchMails(GetMailFromGmail) failed : ", Ex);
}
finally
{
if (m_MailTable.Rows.Count > 0)
{
SaveEmail();
Common.Synch.Factory.LogHelpFactory.log.Info("Save Email Complete");
}
int Sleeptime = Convert.ToInt32(ConfigDetails.GetResourceSleeptime(Constant.InboundProcess));
if (Sleeptime == 0)
{
Sleeptime = 10000;
}
ConfigDetails = null;
try
{
Thread.Sleep(Sleeptime);
}
catch (ThreadAbortException ex)
{
Common.Synch.Factory.LogHelpFactory.log.Info("Error on thread.Sleep:" + ex.Message.ToString());
}
try
{
this.DownLoadCompleteEventArgs(this, new DownLoadCompleteEventArgs("DownLoad Complete"));
}
catch (Exception ex)
{
Common.Synch.Factory.LogHelpFactory.log.Info("Error while invoking the thread:" + ex.Message.ToString());
}
Common.Synch.Factory.LogHelpFactory.log.Info("End Download Mails From Inbox");
}
}
// Log
}
}
catch (Exception ex)
{
Common.Synch.Factory.LogHelpFactory.log.Error(ex.Message.ToString());
}
}
Method to save extracted files in database:
public void GetMaiilboxtotable(MailMessage Message)
{
Common.Synch.Common.DeleteFile(TempPath);
try
{
if (Initalize())
{
if (Message.Attachments != null && Message.AttachmentsCount > 0)
{
for (int Count = 0; Count <= Message.AttachmentsCount - 1; Count++)
{
//if (Message.Attachments[Count].AttachmentAttachmentFileName.Substring(Message.Attachments[Count].AttachmentAttachmentFileName.Length - 3, 3).ToUpper() == "MSG")
//{
// Message.Attachments[Count].SaveAsFile(TempPath + "Temp.msg");
// outlookApp = new OutLook.ApplicationClass();
// MsgMail = (OutLook.MailItem)outlookApp.CreateItemFromTemplate(TempPath + "Temp.msg", Type.Missing);
// GetMaiilboxtotable(MsgMail);
// outlookApp = null;
// MsgMail = null;
//}
if (Message.Attachments[Count].AttachmentFileName.Substring(Message.Attachments[Count].AttachmentFileName.Length - 3, 3).ToUpper() == "ZIP" ||
Message.Attachments[Count].AttachmentFileName.Substring(Message.Attachments[Count].AttachmentFileName.Length - 3, 3).ToUpper() == "XML" ||
Message.Attachments[Count].AttachmentFileName.Substring(Message.Attachments[Count].AttachmentFileName.Length - 3, 3).ToUpper() == "XLS" ||
(Message.Attachments[Count].AttachmentFileName.Substring(Message.Attachments[Count].AttachmentFileName.Length - 3, 3).ToUpper() == "DAT"
&& Message.Attachments[Count].AttachmentFileName.Substring(0, 2).ToUpper() == "HJ") ||
Message.Attachments[Count].AttachmentFileName.Substring(Message.Attachments[Count].AttachmentFileName.Length - 3, 3).ToUpper() == "BIN")
{
if (m_MailTable.Columns.Count <= 0)
{
CreateMailTable();
}
isDeleted = true;
DataRow MailRows = m_MailTable.NewRow();
MailRows["MailID"] = Message.FromEmail;
MailRows["MsgDate"] = Message.ReceivedDate;
MailRows["MsgSubject"] = (null == Message.Subject ? string.Empty : Message.Subject);
MailRows["CreateDateTime"] = DateTime.Now;
//string GwAttachmentName = Message.Attachments[Count].AttachmentFileName;
Message.Attachments[Count].DownloadAttachment();// SaveAsFile(TempPath + Message.Attachments[Count].AttachmentFileName);
MailRows["AttachmentName"] = Message.Attachments[Count].AttachmentFileName;
ConvertBinary ConvertBinary = new ConvertBinary();
MailRows["AttachmentFile"] = ConvertBinary.ConverTOBinaryFile(TempPath + "\\" + Message.Attachments[Count].AttachmentFileName);
ConvertBinary.Dispose();
ConvertBinary = null;
//Marshal.ReleaseComObject(synchGwAttachment);
m_MailTable.Rows.Add(MailRows);
//Log
Common.Synch.Factory.LogHelpFactory.log.Info("Add Mail to MailBox Temp Table -" + Message.Attachments[Count].AttachmentFileName);
}
}
}
}
}
catch (Exception ex)
{
Message = null;
Common.Synch.Factory.LogHelpFactory.log.Info(Message.Subject + ":" + ex.ToString());
}
}
Attachment structure is:
In attachment we are sending bin files of objects and these objects further extracted and saved in database.
Problem:
Some times we are receiving large files, so while extracting service hang and stop processing in between. Could you provide me some suggestions and ideas to overcome this issue.
Please let me know if you need more information, I will try my best to provide.
Thanks in advance.

Answer 1

You are using DataTables.
A DataTable is an in-memory copy of relational data.
So all the data you are going to save have to fit into memory.
If you have a lot of records, big records or both this will not do.
I suggest to:
decompress the file inside a temp folder on local file system
insert into the database one file at a time using a SqlCommand
Use parameters in your INSERT command (ie insert into mytable (col1, col2) values (#col1, # col2))