changed tokenizer interface

CHANGELOG.rst

@@ -32,6 +32,7 @@ Version 2022-dev
 -  allocate threecenter matrices in parallel (#701)
 -  use NDimVector instead of std::vector in vxcgrid. (#703)
 -  Fixing the tutorial (#705)
+-  adapted tokenizer api (#707)
 
 Version 2021.1 (released XX.03.21)
 ==================================

src/libxtp/IndexParser.cc

@@ -37,9 +37,7 @@ namespace xtp {
 std::vector<Index> IndexParser::CreateIndexVector(
     const std::string& Ids) const {
   std::vector<Index> result;
-  tools::Tokenizer tok(Ids, " ,\n\t");
-  std::vector<std::string> results;
-  tok.ToVector(results);
+  std::vector<std::string> results = tools::Tokenizer(Ids, " ,\n\t").ToVector();
   const std::string delimiter = ":";
   for (std::string s : results) {
     if (s.find(delimiter) != std::string::npos) {

src/libxtp/calculators/eanalyze.cc

@@ -17,6 +17,7 @@
 
 // Local private VOTCA includes
 #include "eanalyze.h"
+#include "votca/xtp/qmstate.h"
 
 namespace votca {
 namespace xtp {
@@ -28,13 +29,7 @@ void EAnalyze::ParseOptions(const tools::Property &options) {
   _resolution_spatial = options.get(".resolution_spatial").as<double>();
   _seg_pattern = options.get(".match_pattern").as<std::string>();
 
-  std::string statestrings = options.get(".states").as<std::string>();
-  tools::Tokenizer tok(statestrings, ",\n\t ");
-  std::vector<std::string> string_vec;
-  tok.ToVector(string_vec);
-  for (std::string &state : string_vec) {
-    _states.push_back(QMStateType(state));
-  }
+  _states = options.get(".states").as<std::vector<QMStateType>>();
 
   _doenergy_landscape = options.get(".do_energy_landscape").as<bool>();
 
@@ -248,7 +243,7 @@ void EAnalyze::SiteCorr(const Topology &top, QMStateType state) const {
 
   // Prepare bins
   Index BIN = Index((MAX - MIN) / _resolution_spatial + 0.5) + 1;
-  std::vector<std::vector<double> > histCs;
+  std::vector<std::vector<double>> histCs;
   histCs.resize(BIN);
 
   for (Index i = 0; i < tabcorr.size(); ++i) {

src/libxtp/calculators/ianalyze.cc

@@ -23,6 +23,7 @@
 #include <votca/tools/histogramnew.h>
 
 // Local VOTCA includes
+#include "votca/xtp/qmstate.h"
 #include "votca/xtp/topology.h"
 
 // Local private VOTCA includes
@@ -33,13 +34,7 @@ namespace xtp {
 
 void IAnalyze::ParseOptions(const tools::Property &options) {
 
-  std::string statestrings = options.get(".states").as<std::string>();
-  tools::Tokenizer tok(statestrings, ",\n\t ");
-  std::vector<std::string> string_vec;
-  tok.ToVector(string_vec);
-  for (std::string &state : string_vec) {
-    _states.push_back(QMStateType(state));
-  }
+  _states = options.get(".states").as<std::vector<QMStateType>>();
 
   _resolution_logJ2 = options.get(".resolution_logJ2").as<double>();
   if (options.get(".do_pairtype").as<bool>()) {
@@ -178,7 +173,7 @@ void IAnalyze::IRdependence(Topology &top, QMStateType state) {
 
   // Prepare R bins
   Index pointsR = Index((MAXR - MINR) / _resolution_spatial);
-  std::vector<std::vector<double> > rJ2;
+  std::vector<std::vector<double>> rJ2;
   rJ2.resize(pointsR);
 
   // Loop over distance

src/libxtp/calculators/mapchecker.h

@@ -68,29 +68,14 @@ void MapChecker::ParseOptions(const tools::Property& options) {
 
   _mpfile = options.get(".mp_pdbfile").as<std::string>();
 
-  std::string output_qm =
-      options.ifExistsReturnElseReturnDefault<std::string>(".qm_states", "");
+  _qmstates = options.get(".qm_states").as<std::vector<QMState>>();
 
-  _qmstates = StringToStates(output_qm);
-  std::string output_md =
-      options.ifExistsReturnElseReturnDefault<std::string>(".mp_states", "");
-  _mdstates = StringToStates(output_md);
+  _mdstates = options.get(".mp_states").as<std::vector<QMState>>();
   if (!(_qmstates.empty() && _mdstates.empty())) {
     _mapfile = options.get(".map_file").as<std::string>();
   }
 }
 
-std::vector<QMState> MapChecker::StringToStates(
-    const std::string& states_string) const {
-  std::vector<QMState> result;
-  tools::Tokenizer tok_states(states_string, " \t\n");
-  std::vector<std::string> states = tok_states.ToVector();
-  for (const std::string& s : states) {
-    result.push_back(QMState(s));
-  }
-  return result;
-}
-
 std::string MapChecker::AddStatetoFilename(const std::string& filename,
                                            QMState state) const {
   std::string base = tools::filesystem::GetFileBase(filename);

src/libxtp/calculators/neighborlist.cc

@@ -40,8 +40,7 @@ void Neighborlist::ParseOptions(const tools::Property& options) {
           segprop->get("cutoff").as<double>() * tools::conv::nm2bohr;
 
       tools::Tokenizer tok(types, " ");
-      std::vector<std::string> names;
-      tok.ToVector(names);
+      std::vector<std::string> names = tok.ToVector();
 
       if (names.size() != 2) {
         throw std::runtime_error(

src/libxtp/calculators/vaverage.cc

@@ -46,9 +46,7 @@ std::vector<double> VAverage::ReadOccfile(std::string filename) const {
   while (intt.good()) {
 
     tools::getline(intt, line);
-    std::vector<std::string> split;
-    tools::Tokenizer toker(line, " \t");
-    toker.ToVector(split);
+    std::vector<std::string> split = tools::Tokenizer(line, " \t").ToVector();
     if (!split.size() || split[0] == "#" || split[0].substr(0, 1) == "#") {
       continue;
     }
@@ -79,9 +77,7 @@ std::vector<Rate_Engine::PairRates> VAverage::ReadRatefile(
   while (intt.good()) {
 
     tools::getline(intt, line);
-    std::vector<std::string> split;
-    tools::Tokenizer toker(line, " \t");
-    toker.ToVector(split);
+    std::vector<std::string> split = tools::Tokenizer(line, " \t").ToVector();
     if (!split.size() || split[0] == "#" || split[0].substr(0, 1) == "#") {
       continue;
     }

src/libxtp/esp2multipole.cc

@@ -37,9 +37,7 @@ void Esp2multipole::Initialize(tools::Property& options) {
   _use_mulliken = false;
   _use_CHELPG = false;
   _use_lowdin = false;
-
-  std::string statestring = options.get(key + ".state").as<std::string>();
-  _state.FromString(statestring);
+  _state = options.get(key + ".state").as<QMState>();
 
   _method = options.get(key + ".method").as<std::string>();
 
@@ -70,10 +68,7 @@ void Esp2multipole::Initialize(tools::Property& options) {
       std::vector<tools::Property*> prop_pair =
           options.Select(key + ".constraints.pairs.pair");
       for (tools::Property* prop : prop_pair) {
-        std::string pairstring = prop->as<std::string>();
-        tools::Tokenizer tok(pairstring, "\n\t ,");
-        std::vector<Index> pairvec;
-        tok.ConvertToVector<Index>(pairvec);
+        std::vector<Index> pairvec = prop->as<std::vector<Index>>();
         std::pair<Index, Index> pair;
         pair.first = pairvec[0];
         pair.second = pairvec[1];

src/libxtp/geometry_optimization.cc

@@ -32,8 +32,7 @@ namespace xtp {
 
 void GeometryOptimization::Initialize(tools::Property& options) {
 
-  std::string statestring = options.get(".state").as<std::string>();
-  _opt_state.FromString(statestring);
+  _opt_state = options.get(".state").as<QMState>();
   if (!_opt_state.Type().isExciton()) {
     throw std::runtime_error(
         "At the moment only excitonic states can be optimized");

src/libxtp/gyration.cc

@@ -34,9 +34,7 @@ namespace xtp {
 
 void Density2Gyration::Initialize(tools::Property& options) {
   std::string key = Identify();
-
-  std::string statestring = options.get(key + ".state").as<std::string>();
-  _state.FromString(statestring);
+  _state = options.get(key + ".state").as<QMState>();
   _dostateonly = options.ifExistsReturnElseReturnDefault<bool>(
       key + ".difference_to_groundstate", false);
   _gridsize = options.ifExistsReturnElseReturnDefault<std::string>(

src/libxtp/jobcalculators/iexcitoncl.cc

@@ -56,9 +56,8 @@ std::map<std::string, QMState> IEXCITON::FillParseMaps(
   Tokenizer split_options(Mapstring, ", \t\n");
   std::map<std::string, QMState> type2level;
   for (const std::string& substring : split_options) {
-    std::vector<std::string> segmentpnumber;
     Tokenizer tok(substring, ":");
-    tok.ToVector(segmentpnumber);
+    std::vector<std::string> segmentpnumber = tok.ToVector();
     if (segmentpnumber.size() != 2) {
       throw std::runtime_error("Parser iqm: Segment and exciton labels:" +
                                substring + "are not separated properly");

src/libxtp/jobcalculators/iqm.cc

@@ -124,12 +124,10 @@ void IQM::ParseSpecificOptions(const tools::Property& options) {
 
 std::map<std::string, QMState> IQM::FillParseMaps(
     const std::string& Mapstring) {
-  tools::Tokenizer split_options(Mapstring, ", \t\n");
   std::map<std::string, QMState> type2level;
-  for (const std::string& substring : split_options) {
-    std::vector<std::string> segmentpnumber;
-    tools::Tokenizer tok(substring, ":");
-    tok.ToVector(segmentpnumber);
+  for (const std::string& substring : tools::Tokenizer(Mapstring, ", \t\n")) {
+    std::vector<std::string> segmentpnumber =
+        tools::Tokenizer(substring, ":").ToVector();
     if (segmentpnumber.size() != 2) {
       throw std::runtime_error("Parser iqm: Segment and exciton labels:" +
                                substring + "are not separated properly");
@@ -143,7 +141,6 @@ std::map<std::string, QMState> IQM::FillParseMaps(
 
 void IQM::addLinkers(std::vector<const Segment*>& segments,
                      const Topology& top) {
-  std::vector<QMState> result;
   const Segment* seg1 = segments[0];
   const Segment* seg2 = segments[1];
   std::vector<const Segment*> segmentsInMolecule =

src/libxtp/jobcalculators/qmmm.cc

@@ -43,11 +43,8 @@ void QMMM::ParseSpecificOptions(const tools::Property& options) {
   _regions_def = options.get(".regions");
   _regions_def.add("mapfile", _mapfile);
 
-  std::string states = options.get(".write_parse.states").as<std::string>();
-  tools::Tokenizer tok(states, " ,;\n\t");
-  for (const auto& s : tok.ToVector()) {
-    _states.push_back(QMState(s));
-  }
+  _states = options.get(".write_parse.states").as<std::vector<QMState>>();
+
   bool groundstate_found = std::any_of(
       _states.begin(), _states.end(),
       [](const QMState& state) { return state.Type() == QMStateType::Gstate; });

src/libxtp/jobtopology.cc

@@ -87,7 +87,7 @@ void JobTopology::BuildRegions(const Topology& top, tools::Property options) {
   SortRegionsDefbyId(regions_def);
   ModifyOptionsByJobFile(regions_def);
 
-  std::vector<std::vector<SegId> > region_seg_ids =
+  std::vector<std::vector<SegId>> region_seg_ids =
       PartitionRegions(regions_def, top);
 
   // around this point the whole jobtopology will be centered
@@ -145,7 +145,7 @@ void JobTopology::ShiftPBC(const Topology& top, const Eigen::Vector3d& center,
 
 void JobTopology::CreateRegions(
     const tools::Property& options, const Topology& top,
-    const std::vector<std::vector<SegId> >& region_seg_ids) {
+    const std::vector<std::vector<SegId>>& region_seg_ids) {
   std::string mapfile =
       options.ifExistsReturnElseThrowRuntimeError<std::string>("mapfile");
   std::vector<const tools::Property*> regions_def = options.Select("region");
@@ -222,12 +222,12 @@ void JobTopology::WriteToPdb(std::string filename) const {
   writer.Close();
 }
 
-std::vector<std::vector<SegId> > JobTopology::PartitionRegions(
+std::vector<std::vector<SegId>> JobTopology::PartitionRegions(
     const std::vector<tools::Property*>& regions_def,
     const Topology& top) const {
 
   std::vector<Index> explicitly_named_segs_per_region;
-  std::vector<std::vector<SegId> > segids_per_region;
+  std::vector<std::vector<SegId>> segids_per_region;
   std::vector<bool> processed_segments =
       std::vector<bool>(top.Segments().size(), false);
   for (const tools::Property* region_def : regions_def) {
@@ -239,12 +239,7 @@ std::vector<std::vector<SegId> > JobTopology::PartitionRegions(
     }
     std::vector<SegId> seg_ids;
     if (region_def->exists("segments")) {
-      std::string seg_ids_string =
-          region_def->get("segments").as<std::string>();
-      tools::Tokenizer tok(seg_ids_string, " \n\t");
-      for (const std::string& seg_id_string : tok.ToVector()) {
-        seg_ids.push_back(SegId(seg_id_string));
-      }
+      seg_ids = region_def->get("segments").as<std::vector<SegId>>();
       for (const SegId& seg_id : seg_ids) {
         if (seg_id.Id() > Index(top.Segments().size() - 1)) {
           throw std::runtime_error("Segment id is not in topology");

src/libxtp/md2qmengine.cc

@@ -49,9 +49,8 @@ void Md2QmEngine::CheckMappingFile(tools::Property& topology_map) const {
 
       std::vector<std::string> atomnames_seg;
       for (tools::Property* frag : fragments) {
-        std::string mdatoms = frag->get("mdatoms").as<std::string>();
-        tools::Tokenizer tok_md_atoms(mdatoms, " \t\n");
-        std::vector<std::string> atomnames = tok_md_atoms.ToVector();
+        std::vector<std::string> atomnames =
+            frag->get("mdatoms").as<std::vector<std::string>>();
         atomnames_seg.insert(atomnames_seg.end(), atomnames.begin(),
                              atomnames.end());
       }
@@ -133,13 +132,13 @@ Topology Md2QmEngine::map(const csg::Topology& top) const {
   xtptop.setBox(top.getBox() * tools::conv::nm2bohr, top.getBoxType());
 
   // which segmentname does an atom belong to molname atomid
-  std::map<std::string, std::map<Index, std::string> > MolToSegMap;
+  std::map<std::string, std::map<Index, std::string>> MolToSegMap;
 
   // which atomids belong to molname
-  std::map<std::string, std::vector<Index> > MolToAtomIds;
+  std::map<std::string, std::vector<Index>> MolToAtomIds;
 
   // names of segments in one molecule;
-  std::map<std::string, std::vector<std::string> > SegsinMol;
+  std::map<std::string, std::vector<std::string>> SegsinMol;
 
   std::string molkey = "topology.molecules.molecule";
   std::vector<tools::Property*> molecules = topology_map.Select(molkey);
@@ -155,10 +154,8 @@ Topology Md2QmEngine::map(const csg::Topology& top) const {
       std::string fragkey = "fragments.fragment";
       std::vector<tools::Property*> fragments = seg->Select(fragkey);
       for (tools::Property* frag : fragments) {
-        std::string mdatoms = frag->get("mdatoms").as<std::string>();
-        tools::Tokenizer tok_md_atoms(mdatoms, " \t\n");
-        std::vector<std::string> atomnames;
-        tok_md_atoms.ToVector(atomnames);
+        std::vector<std::string> atomnames =
+            frag->get("mdatoms").as<std::vector<std::string>>();
         for (const std::string& atomname : atomnames) {
           tools::Tokenizer tok_atom_name(atomname, ":");
           std::vector<std::string> entries = tok_atom_name.ToVector();

src/libxtp/numerical_integration/vxc_potential.cc

@@ -44,8 +44,9 @@ double Vxc_Potential<Grid>::getExactExchange(const std::string& functional) {
 
   double exactexchange = 0.0;
   Vxc_Functionals map;
-  tools::Tokenizer tok(functional, " ");
-  std::vector<std::string> functional_names = tok.ToVector();
+
+  std::vector<std::string> functional_names =
+      tools::Tokenizer(functional, " ").ToVector();
 
   if (functional_names.size() > 2) {
     throw std::runtime_error("Too many functional names");
@@ -79,9 +80,8 @@ template <class Grid>
 void Vxc_Potential<Grid>::setXCfunctional(const std::string& functional) {
 
   Vxc_Functionals map;
-  std::vector<std::string> strs;
-  tools::Tokenizer tok(functional, " ,\n\t");
-  tok.ToVector(strs);
+  std::vector<std::string> strs =
+      tools::Tokenizer(functional, " ,\n\t").ToVector();
   xfunc_id = 0;
   _use_separate = false;
   cfunc_id = 0;

src/libxtp/progressobserver.cc

@@ -225,8 +225,8 @@ void ProgObserver<JobContainer>::InitCmdLineOpts(
     _restartMode = true;
   }
 
-  tools::Tokenizer toker(restartPattern, "(,)");
-  std::vector<std::string> patterns = toker.ToVector();
+  std::vector<std::string> patterns =
+      tools::Tokenizer(restartPattern, "(,)").ToVector();
 
   std::string category = "";
   for (const std::string &pattern : patterns) {

src/libxtp/qmpackage.cc

@@ -136,8 +136,7 @@ std::vector<std::string> QMPackage::GetLineAndSplit(
   std::string line;
   tools::getline(input_file, line);
   boost::trim(line);
-  tools::Tokenizer tok(line, separators);
-  return tok.ToVector();
+  return tools::Tokenizer(line, separators).ToVector();
 }
 
 std::string QMPackage::FindDefaultsFile() const {