IBR-DTNSuite  0.12
EMailImapService.cpp
Go to the documentation of this file.
1 /*
2  * EMailImapService.cpp
3  *
4  * Copyright (C) 2013 IBR, TU Braunschweig
5  *
6  * Written-by: Björn Gernert <mail@bjoern-gernert.de>
7  *
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  *
20  */
21 
22 #include "net/EMailImapService.h"
23 #include "net/BundleReceivedEvent.h"
24 #include "core/BundleCore.h"
25 #include "core/BundleEvent.h"
26 #include "ibrdtn/data/BundleBuilder.h"
27 #include "routing/RequeueBundleEvent.h"
28 
29 #include <ibrcommon/Logger.h>
30 #include <vmime/platforms/posix/posixHandler.hpp>
31 
32 namespace dtn
33 {
34  namespace net
35  {
36  bool EMailImapService::_run(false);
37 
38  EMailImapService& EMailImapService::getInstance()
39  {
40  static EMailImapService instance;
41  return instance;
42  }
43 
44  EMailImapService::EMailImapService()
45  : _config(daemon::Configuration::getInstance().getEMail()),
46  _certificateVerifier(vmime::create<vmime::security::cert::defaultCertificateVerifier>())
47  {
48  // Load certificates
49  loadCerificates();
50 
51  try {
52  // Initialize vmime engine
53  vmime::platform::setHandler<vmime::platforms::posix::posixHandler>();
54 
55  // Set IMAP-Server for outgoing emails (use SSL or not)
56  std::string url;
57  if(_config.imapUseSSL())
58  {
59  url = "imaps://" + _config.getImapServer();
60  }else{
61  url = "imap://" + _config.getImapServer();
62  }
63  vmime::ref<vmime::net::session> session =
64  vmime::create<vmime::net::session>();
65 
66  // Create an instance of the store service
67  _store = session->getStore(vmime::utility::url(url));
68 
69  // Set username, password and port
70  _store->setProperty("auth.username", _config.getImapUsername());
71  _store->setProperty("auth.password", _config.getImapPassword());
72  _store->setProperty("server.port", _config.getImapPort());
73 
74  // Enable TLS
75  if(_config.imapUseTLS())
76  {
77  _store->setProperty("connection.tls", true);
78  _store->setProperty("connection.tls.required", true);
79  }
80 
81  // Add certificate verifier
82  if(_config.imapUseTLS() || _config.imapUseSSL())
83  {
84  _store->setCertificateVerifier(_certificateVerifier);
85  }
86 
87  // Handle timeouts
88  _store->setTimeoutHandlerFactory(vmime::create<TimeoutHandlerFactory>());
89 
90  // Create the folder path
91  std::vector<std::string> folderPath = _config.getImapFolder();
92  if(!folderPath.empty())
93  {
94  for(std::vector<std::string>::iterator it = folderPath.begin() ; it != folderPath.end(); ++it)
95  _path /= vmime::net::folder::path::component(*it);
96  }
97 
98  // Set mutex
99  _threadMutex.enter();
100 
101  // Start thread
102  _run = true;
103  this->start();
104  } catch (vmime::exception &e) {
105  IBRCOMMON_LOGGER(error) << "EMail Convergence Layer: Unable to create IMAP service: " << e.what() << IBRCOMMON_LOGGER_ENDL;
106  }
107  }
108 
109  EMailImapService::~EMailImapService()
110  {
111  this->stop();
112  this->join();
113 
114  // Delete remaining processed tasks
115  {
116  ibrcommon::Mutex l(_processedTasksMutex);
117  while(!_processedTasks.empty())
118  {
119  EMailSmtpService::Task *t = _processedTasks.front();
120  _processedTasks.pop_front();
121  delete t;
122  t = NULL;
123  }
124  }
125  }
126 
127  void EMailImapService::loadCerificates()
128  {
129  std::vector<std::string> certPath;
130  // Load CA certificates
131  std::vector<vmime::ref <vmime::security::cert::X509Certificate> > ca;
132  certPath = _config.getTlsCACerts();
133  if(!certPath.empty()) {
134  for(std::vector<std::string>::iterator it = certPath.begin() ; it != certPath.end(); ++it)
135  {
136  try {
137  ca.push_back(loadCertificateFromFile((*it)));
138  }catch(InvalidCertificate &e) {
139  IBRCOMMON_LOGGER(error) << "EMail Convergence Layer: Certificate load error: " << e.what() << IBRCOMMON_LOGGER_ENDL;
140  }
141  }
142  _certificateVerifier->setX509RootCAs(ca);
143  }
144 
145  // Load user certificates
146  std::vector<vmime::ref <vmime::security::cert::X509Certificate> > user;
147  certPath = _config.getTlsUserCerts();
148  if(!certPath.empty()) {
149  for(std::vector<std::string>::iterator it = certPath.begin() ; it != certPath.end(); ++it)
150  {
151  try {
152  user.push_back(loadCertificateFromFile((*it)));
153  }catch(InvalidCertificate &e) {
154  IBRCOMMON_LOGGER(error) << "EMail Convergence Layer: Certificate load error: " << e.what() << IBRCOMMON_LOGGER_ENDL;
155  }
156  }
157  _certificateVerifier->setX509TrustedCerts(user);
158  }
159  }
160 
161  vmime::ref<vmime::security::cert::X509Certificate> EMailImapService::loadCertificateFromFile(const std::string &path)
162  {
163  std::ifstream certFile;
164  certFile.open(path.c_str(), std::ios::in | std::ios::binary);
165  if(!certFile)
166  {
167  throw(InvalidCertificate("Unable to find certificate at \"" + path + "\""));
168  return NULL;
169  }
170 
171  vmime::utility::inputStreamAdapter is(certFile);
172  vmime::ref<vmime::security::cert::X509Certificate> cert;
173  cert = vmime::security::cert::X509Certificate::import(is);
174  if(cert != NULL)
175  {
176  return cert;
177  }else{
178  throw(InvalidCertificate("The certificate at \"" + path + "\" does not seem to be PEM or DER encoded"));
179  return NULL;
180  }
181  }
182 
183  void EMailImapService::run() throw ()
184  {
185  while(true)
186  {
187  _threadMutex.enter();
188 
189  if(!_run)
190  break;
191 
192  try {
193  IBRCOMMON_LOGGER(info) << "EMail Convergence Layer: Looking for new bundles via IMAP" << IBRCOMMON_LOGGER_ENDL;
194  queryServer();
195  }catch(vmime::exception &e) {
196  if(_run)
197  IBRCOMMON_LOGGER(error) << "EMail Convergence Layer: Error during IMAP fetch operation: " << e.what() << IBRCOMMON_LOGGER_ENDL;
198  }catch(IMAPException &e) {
199  if(_run)
200  IBRCOMMON_LOGGER(error) << "EMail Convergence Layer: IMAP error: " << e.what() << IBRCOMMON_LOGGER_ENDL;
201  }
202  }
203  }
204 
205  void EMailImapService::__cancellation() throw ()
206  {
207  _run = false;
208  disconnect();
209  _threadMutex.leave();
210  }
211 
212  void EMailImapService::storeProcessedTask(EMailSmtpService::Task *t)
213  {
214  ibrcommon::Mutex l(_processedTasksMutex);
215  _processedTasks.push_back(t);
216  }
217 
218  void EMailImapService::fetchMails() {
219  if(_run)
220  _threadMutex.leave();
221  }
222 
223  bool EMailImapService::TimeoutHandler::isTimeOut()
224  {
225  if(!_run)
226  return true;
227 
228  return (getTime() >= last +
229  daemon::Configuration::getInstance().getEMail().getImapConnectionTimeout());
230  }
231 
232  void EMailImapService::TimeoutHandler::resetTimeOut()
233  {
234  last = getTime();
235  }
236 
237  bool EMailImapService::TimeoutHandler::handleTimeOut()
238  {
239  // true = continue, false = cancel
240  return false;
241  }
242 
243  void EMailImapService::connect()
244  {
245  // Connect to IMAP Server
246  if(!isConnected())
247  {
248  try {
249  _store->connect();
250  // Get working folder
251  if(!_path.isEmpty())
252  {
253  _folder = _store->getFolder(_path);
254  } else {
255  _folder = _store->getDefaultFolder();
256  }
257  // Open with read-write access
258  _folder->open(vmime::net::folder::MODE_READ_WRITE, true);
259  }catch(vmime::exceptions::certificate_verification_exception&) {
260  throw IMAPException("Cannot verify certificate against trusted certificates");
261  }catch(vmime::exceptions::authentication_error&) {
262  throw IMAPException("Authentication error (username/password correct?)");
263  }catch(vmime::exceptions::connection_error&) {
264  throw IMAPException("Unable to connect to the IMAP server");
265  }catch(vmime::exception &e) {
266  throw IMAPException(e.what());
267  }
268  }
269  }
270 
271  bool EMailImapService::isConnected()
272  {
273  return _store->isConnected();
274  }
275 
276  void EMailImapService::disconnect()
277  {
278  if(isConnected())
279  {
280  if(_folder->isOpen())
281  _folder->close(_config.imapPurgeMail());
282  _store->disconnect();
283  }
284  }
285 
286  void EMailImapService::queryServer()
287  {
288  // Check connection
289  if(!isConnected())
290  connect();
291 
292  // Get all unread messages
293  std::vector<vmime::ref<vmime::net::message> > allMessages;
294  try {
295  allMessages = _folder->getMessages();
296  _folder->fetchMessages(allMessages,
297  vmime::net::folder::FETCH_FLAGS | vmime::net::folder::FETCH_FULL_HEADER | vmime::net::folder::FETCH_UID);
298  }catch(vmime::exception&) {
299  // No messages found (folder empty)
300  }
301  for(std::vector<vmime::ref<vmime::net::message> >::iterator it =
302  allMessages.begin(); it != allMessages.end(); ++it)
303  {
304  const int flags = (*it).get()->getFlags();
305 
306  if(!flags & vmime::net::message::FLAG_SEEN) {
307  // Looking for new bundles for me
308  try {
309  try {
310  IBRCOMMON_LOGGER(info) << "EMail Convergence Layer: Generating bundle from mail (" << it->get()->getUniqueId().c_str() << ")" << IBRCOMMON_LOGGER_ENDL;
311  generateBundle((*it));
312  }catch(IMAPException &e){
313  try {
314  returningMailCheck((*it));
315  }catch(BidNotFound&) {
316  IBRCOMMON_LOGGER(warning) << "EMail Convergence Layer: " << e.what() << IBRCOMMON_LOGGER_ENDL;
317  }
318  }
319  }catch(vmime::exceptions::no_such_field &e) {
320  IBRCOMMON_LOGGER(warning) << "EMail Convergence Layer: Mail " << (*it)->getUniqueId() << " has no subject, skipping" << IBRCOMMON_LOGGER_ENDL;
321  }
322 
323  // Set mail flag to 'seen'
324  (*it)->setFlags(vmime::net::message::FLAG_SEEN, vmime::net::message::FLAG_MODE_SET);
325 
326  // Purge storage if enabled
327  if(_config.imapPurgeMail())
328  {
329  (*it)->setFlags(vmime::net::message::FLAG_DELETED, vmime::net::message::FLAG_MODE_SET);
330  }
331  } else {
332  continue;
333  }
334  }
335 
336  // Delete old tasks
337  {
338  ibrcommon::Mutex l(_processedTasksMutex);
339  std::list<EMailSmtpService::Task*>::iterator it = _processedTasks.begin();
340  while(it != _processedTasks.end())
341  {
342  if(!(*it)->checkForReturningMail()) {
343  dtn::net::BundleTransfer job = (*it)->getJob();
344  job.complete();
345  delete *it;
346  *it = NULL;
347  _processedTasks.erase(it++);
348  }
349  }
350  }
351 
352  disconnect();
353  }
354 
355  void EMailImapService::generateBundle(vmime::ref<vmime::net::message> &msg)
356  {
357 
358  // Create new Bundle
359  dtn::data::Bundle newBundle;
360 
361  // Clear all blocks
362  newBundle.clear();
363 
364  // Get header of mail
365  vmime::ref<const vmime::header> header = msg->getHeader();
366  std::string mailID = " (ID: " + msg->getUniqueId() + ")";
367 
368  // Check EMailCL version
369  try {
370  int version = toInt(header->findField("Bundle-EMailCL-Version")->getValue()->generate());
371  if(version != 1)
372  throw IMAPException("Wrong EMailCL version found in mail" + mailID);
373  }catch(vmime::exceptions::no_such_field&) {
374  throw IMAPException("Field \"Bundle-EMailCL-Version\" not found in mail" + mailID);
375  }catch(InvalidConversion&) {
376  throw IMAPException("Field \"Bundle-EMailCL-Version\" wrong formatted in mail" + mailID);
377  }
378 
379  try {
380  newBundle.procflags = toInt(header->findField("Bundle-Flags")->getValue()->generate());
381  newBundle.destination = header->findField("Bundle-Destination")->getValue()->generate();
382  newBundle.source = dtn::data::EID(header->findField("Bundle-Source")->getValue()->generate());
383  newBundle.reportto = header->findField("Bundle-Report-To")->getValue()->generate();
384  newBundle.custodian = header->findField("Bundle-Custodian")->getValue()->generate();
385  newBundle.timestamp = toInt(header->findField("Bundle-Creation-Time")->getValue()->generate());
386  newBundle.sequencenumber = toInt(header->findField("Bundle-Sequence-Number")->getValue()->generate());
387  newBundle.lifetime = toInt(header->findField("Bundle-Lifetime")->getValue()->generate());
388  }catch(vmime::exceptions::no_such_field&) {
389  throw IMAPException("Missing bundle headers in mail" + mailID);
390  }catch(InvalidConversion&) {
391  throw IMAPException("Wrong formatted bundle headers in mail" + mailID);
392  }
393 
394  // If bundle is a fragment both fields need to be set
395  try {
396  newBundle.fragmentoffset = toInt(header->findField("Bundle-Fragment-Offset")->getValue()->generate());
397  newBundle.appdatalength = toInt(header->findField("Bundle-Total-Application-Data-Unit-Length")->getValue()->generate());
398  }catch(vmime::exceptions::no_such_field&) {
399  newBundle.fragmentoffset = 0;
400  newBundle.appdatalength = 0;
401  }catch(InvalidConversion&) {
402  throw IMAPException("Wrong formatted fragment offset in mail" + mailID);
403  }
404 
405  //Check if bundle is already in the storage
406  try {
407  if(dtn::core::BundleCore::getInstance().getRouter().isKnown(newBundle))
408  throw IMAPException("Bundle in mail" + mailID + " already processed");
409  } catch (dtn::storage::NoBundleFoundException&) {
410  // Go ahead
411  }
412 
413  // Validate the primary block
414  try {
415  dtn::core::BundleCore::getInstance().validate((dtn::data::PrimaryBlock)newBundle);
416  }catch(dtn::data::Validator::RejectedException&) {
417  throw IMAPException("Bundle in mail" + mailID + " was rejected by validator");
418  }
419 
420  // Get names of additional blocks
421  std::vector<vmime::ref<vmime::headerField> > additionalBlocks =
422  header.constCast<vmime::header>()->findAllFields("Bundle-Additional-Block");
423  std::vector<std::string> additionalBlockNames;
424  for(std::vector<vmime::ref<vmime::headerField> >::iterator it = additionalBlocks.begin();
425  it != additionalBlocks.end(); ++it)
426  {
427  additionalBlockNames.push_back((*it)->getValue()->generate());
428  }
429 
430  // Get all attachments
431  std::vector<vmime::ref<const vmime::attachment> > attachments =
432  vmime::attachmentHelper::findAttachmentsInMessage(msg->getParsedMessage());
433 
434  // Extract payload block info
435  size_t payloadFlags;
436  std::string payloadName;
437  try {
438  payloadFlags = toInt(header->findField("Bundle-Payload-Flags")->getValue()->generate());
439  // Payload length will be calculated automatically
440  //length = toInt(header->findField("Bundle-Payload-Block-Length")->getValue()->generate());
441  payloadName = header->findField("Bundle-Payload-Data-Name")->getValue()->generate();
442  }catch(vmime::exceptions::no_such_field&) {
443  // No payload block there
444  }catch(InvalidConversion&) {
445  throw IMAPException("Wrong formatted payload flags in mail" + mailID);
446  }
447 
448  // Create new bundle builder
449  dtn::data::BundleBuilder builder(newBundle);
450 
451  // Download attachments
452  for(std::vector<vmime::ref<const vmime::attachment> >::iterator it = attachments.begin(); it != attachments.end(); ++it)
453  {
454  if((*it)->getName().getBuffer() == payloadName && !payloadName.empty()) {
455  dtn::data::PayloadBlock &pb = newBundle.push_back<dtn::data::PayloadBlock>();
456  // Set data
457  ibrcommon::BLOB::Reference ref = pb.getBLOB();
458  ibrcommon::BLOB::iostream payloadData = ref.iostream();
459  vmime::utility::outputStreamAdapter data((*payloadData));
460  (*it)->getData()->extract(data);
461  // Set flags
462  setProcFlags(pb, payloadFlags);
463  continue;
464  }
465 
466  // If current attachment name is contained in additional attachment list
467  if(std::find(additionalBlockNames.begin(), additionalBlockNames.end(), (*it)->getName().getBuffer()) != additionalBlockNames.end())
468  {
469  // Search for the block type
470  block_t blockType;
471  try {
472  blockType = toInt((*it)->getHeader()->findField("Block-Type")->getValue()->generate());
473  }catch(vmime::exceptions::no_such_field&) {
474  throw IMAPException("Block type not found in attachment of mail" + mailID);
475  }
476 
477  // Search for processing flags
478  size_t flags;
479  try {
480  flags = toInt((*it)->getHeader()->findField("Block-Processing-Flags")->getValue()->generate());
481  }catch(vmime::exceptions::no_such_field&) {
482  throw IMAPException("Missing block processing flags in extension attachment of mail" + mailID);
483  }catch(InvalidConversion&) {
484  throw IMAPException("Wrong formatted processing flags in extension attachment of mail" + mailID);
485  }
486 
487  // Create a block object
488  dtn::data::Block &block = builder.insert(blockType, flags);
489 
490  // Add EIDs
491  try {
492  addEIDList(block, (*it));
493  }catch(InvalidConversion&) {
494  throw IMAPException("Wrong formatted EID list in extension attachment of mail" + mailID);
495  }
496 
497  // Get payload of current block
498  std::stringstream ss;
499  vmime::utility::outputStreamAdapter data(ss);
500  (*it)->getData()->extract(data);
501 
502  block.deserialize(ss, ss.str().length());
503  }
504  }
505 
506  // Validate whole bundle
507  try {
508  dtn::core::BundleCore::getInstance().validate(newBundle);
509  }catch(dtn::data::Validator::RejectedException&) {
510  throw IMAPException("Bundle in mail" + mailID + " was rejected by validator");
511  }
512 
513  // Raise default bundle received event
514  dtn::net::BundleReceivedEvent::raise(newBundle.source, newBundle, false);
515  }
516 
517  void EMailImapService::returningMailCheck(vmime::ref<vmime::net::message> &msg)
518  {
519  dtn::data::BundleID bid;
520 
521  bool bidFound = false;
522 
523  std::string s;
524  vmime::utility::outputStreamStringAdapter out(s);
525 
526  vmime::messageParser mp(msg->getParsedMessage());
527 
528  for(int i = 0; i < mp.getTextPartCount(); ++i)
529  {
530  s.clear();
531  mp.getTextPartAt(i)->getText()->extract(out);
532 
533  try {
534  bid = extractBID(s);
535  bidFound = true;
536  break;
537  }catch(InvalidConversion&) {}
538  }
539 
540  if(!bidFound)
541  {
542  for(int i = 0; i < mp.getAttachmentCount(); ++i)
543  {
544  s.clear();
545  mp.getAttachmentAt(i)->getData()->extract(out);
546 
547  try {
548  bid = extractBID(s);
549  bidFound = true;
550  break;
551  }catch(InvalidConversion&) {}
552  }
553  }
554 
555  {
556  ibrcommon::Mutex l(_processedTasksMutex);
557  for(std::list<EMailSmtpService::Task*>::iterator iter = _processedTasks.begin(); iter != _processedTasks.end(); ++iter)
558  {
559  if((*iter)->getJob().getBundle() == bid)
560  {
561  dtn::routing::RequeueBundleEvent::raise((*iter)->getNode().getEID(), bid);
562  _processedTasks.erase(iter);
563  break;
564  }
565  }
566  }
567 
568  if(!bidFound)
569  throw BidNotFound("Found no bundle ID");
570  }
571 
572  dtn::data::BundleID EMailImapService::extractBID(const std::string &message) {
573  try {
574  dtn::data::BundleID ret;
575 
576  ret.source = searchString("Bundle-Source: ", message);
577  ret.timestamp = toInt(searchString("Bundle-Creation-Time: ", message));
578  ret.sequencenumber = toInt(searchString("Bundle-Sequence-Number: ", message));
579  try {
580  ret.fragmentoffset = toInt(searchString("Bundle-Fragment-Offset: ", message));
581  ret.setFragment(true);
582  }catch(...){}
583 
584  return ret;
585  }catch(...) {
586  throw InvalidConversion("No bundle ID was found");
587  }
588  }
589 
590  std::string EMailImapService::searchString(const std::string &search, const std::string &s)
591  {
592  size_t start = 0, end = 0;
593  start = s.find(search);
594  if(start == std::string::npos)
595  throw StringNotFound("Unable to find the string");
596  start = start + search.length();
597 
598  end = s.find("\r\n", start);
599  if(start == std::string::npos)
600  throw StringNotFound("Unable to find the string");
601 
602  return s.substr(start, end-start);
603  }
604 
605  int EMailImapService::toInt(std::string s)
606  {
607  std::stringstream ss(s);
608  int ret;
609  ss >> ret;
610  if (ss.rdbuf()->in_avail() == 0) return ret;
611  else throw InvalidConversion("Invalid integer " + s + ".");
612  }
613 
614  std::string EMailImapService::toString(int i)
615  {
616  std::stringstream ss;
617  ss << i;
618  return ss.str();
619  }
620 
621  void EMailImapService::addEIDList(dtn::data::Block &block, vmime::utility::ref<const vmime::attachment> &attachment)
622  {
623  std::vector<vmime::ref<vmime::headerField> > eidFiels =
624  attachment->getHeader().constCast<vmime::header>()->findAllFields("Block-EID-Reference");
625  try {
626  for(std::vector<vmime::ref<vmime::headerField> >::iterator it = eidFiels.begin(); it != eidFiels.end(); ++it)
627  block.addEID(dtn::data::EID((*it)->getValue()->generate()));
628  }catch(vmime::exceptions::no_such_field&) {
629  throw InvalidConversion("Invalid EID field.");
630  }
631  }
632 
633  void EMailImapService::setProcFlags(dtn::data::Block &block, size_t &flags)
634  {
635  if(flags & dtn::data::Block::REPLICATE_IN_EVERY_FRAGMENT)
636  block.set(dtn::data::Block::REPLICATE_IN_EVERY_FRAGMENT, true);
637  if(flags & dtn::data::Block::TRANSMIT_STATUSREPORT_IF_NOT_PROCESSED)
638  block.set(dtn::data::Block::TRANSMIT_STATUSREPORT_IF_NOT_PROCESSED, true);
639  if(flags & dtn::data::Block::DELETE_BUNDLE_IF_NOT_PROCESSED)
640  block.set(dtn::data::Block::DELETE_BUNDLE_IF_NOT_PROCESSED, true);
641  if(flags & dtn::data::Block::LAST_BLOCK)
642  block.set(dtn::data::Block::LAST_BLOCK, true);
643  if(flags & dtn::data::Block::DISCARD_IF_NOT_PROCESSED)
644  block.set(dtn::data::Block::DISCARD_IF_NOT_PROCESSED, true);
645  if(flags & dtn::data::Block::FORWARDED_WITHOUT_PROCESSED)
646  block.set(dtn::data::Block::FORWARDED_WITHOUT_PROCESSED, true);
647  if(flags & dtn::data::Block::BLOCK_CONTAINS_EIDS)
648  block.set(dtn::data::Block::BLOCK_CONTAINS_EIDS, true);
649  }
650  }
651 }