Support for external generators in RTE Model (continue) (#1335)

* Support for external generators in RTE Model (continue)

Co-authored-by: Evgueni Driouk <edriouk@arm.com>

libs/rtemodel/include/RteGenerator.h

@@ -64,10 +64,11 @@ class RteGenerator : public RteItem
 
   /**
   * @brief get expanded generator executable command
+  * @param target pointer to RteTarget
   * @param hostType host type to match, empty to match current host
   * @return generator command for specified host type
  */
-  virtual std::string GetExecutable(const std::string& hostType = EMPTY_STRING) const;
+   std::string GetExecutable(RteTarget* target, const std::string& hostType = EMPTY_STRING) const;
 
   /**
    * @brief get item containing command line arguments
@@ -139,19 +140,21 @@ class RteGenerator : public RteItem
 
  /**
  * @brief get all arguments as vector for the given host type
+ * @param target pointer to RteTarget
  * @param hostType host type, empty to match current host
  * @param dryRun include dry-run arguments
  * @return vector of arguments consisting of switch and value in pairs
 */
-  std::vector<std::pair<std::string, std::string> > GetExpandedArguments(const std::string& hostType = EMPTY_STRING, bool dryRun = false) const;
+  std::vector<std::pair<std::string, std::string> > GetExpandedArguments(RteTarget* target,const std::string& hostType = EMPTY_STRING, bool dryRun = false) const;
 
  /**
    * @brief get full command line with arguments and expanded key sequences for specified target
+   * @param target pointer to RteTarget
    * @param hostType host type, empty to match current host
    * @param dryRun include dry-run arguments
    * @return expanded command line with arguments, properly quoted
   */
-  std::string GetExpandedCommandLine(const std::string& hostType = EMPTY_STRING, bool dryRun = false) const;
+  std::string GetExpandedCommandLine(RteTarget* target, const std::string& hostType = EMPTY_STRING, bool dryRun = false) const;
 
   /**
    * @brief get absolute path to gpdsc file for specified target

libs/rtemodel/include/RteItem.h

@@ -593,11 +593,14 @@ class RteItem : public XmlTreeItem<RteItem>
   virtual int GetMaxInstances() const;
 
   /**
-   * @brief expands key sequences ("@L", "%L", etc.) in the supplied string.
+   * @brief expands key sequences ("@L", "%L", etc.) or access sequences in the supplied string.
    * @param str string to expand
+   * @param bUseAccessSequences expand access sequences instead of key sequences, default is false
+   * @param context pointer to RteItem representing expansion context (optional)
    * @return expanded string
   */
-  virtual std::string ExpandString(const std::string& str) const;
+  virtual std::string ExpandString(const std::string& str,
+    bool bUseAccessSequences = false, RteItem* context = nullptr) const;
 
   /**
    * @brief get item's description

libs/rtemodel/include/RteItemBuilder.h

@@ -66,7 +66,7 @@ class RteItemBuilder : public XmlTreeItemBuilder<RteItem>
   */
   void SetPackageState(PackageState packState) { m_packState = packState; }
 
-private:
+protected:
   RteItem* m_rootParent;
   PackageState m_packState;
 

libs/rtemodel/include/RteKernel.h

@@ -13,6 +13,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 /******************************************************************************/
+#include "RteItemBuilder.h"
 #include "RteModel.h"
 #include "RteProject.h"
 #include "RteTarget.h"
@@ -260,18 +261,23 @@ class RteKernel
   std::string GetPdscFileFromPath(const XmlItem& attributes, const std::string& cprjPath, std::string& packId);
 
   /**
-   * @brief create a smart pointer holding an XMLTree pointer to parse XML files
+   * @brief create a smart pointer holding an XMLTree pointer to parse XML or YAML files
    * @param itemBuilder pointer to IXmlItemBuilder item factory
+   * @param ext file extension to define which parser is required: XML (default) or YAML (".yml" or ".yaml")
    * @return a std::unique_ptr object holding an XMLTree-derived pointer which is nullptr in the default implementation
   */
-  virtual std::unique_ptr<XMLTree> CreateUniqueXmlTree(IXmlItemBuilder* itemBuilder = nullptr) const;
+  virtual std::unique_ptr<XMLTree> CreateUniqueXmlTree(IXmlItemBuilder* itemBuilder = nullptr, const std::string& ext= RteUtils::EMPTY_STRING) const;
 
   /**
-   * @brief create a smart pointer holding a YmlTree pointer to parse YAML files
-   * @param itemBuilder pointer to IXmlItemBuilder item factory
-   * @return a std::unique_ptr object holding a YmlTree-derived pointer which is nullptr in the default implementation
+   * @brief create a smart pointer holding an IXmlItemBuilder pointer to be used by XMLTree or YmlTree
+   * @param rootParent pointer to RteItem to be parent for root items (optional)
+   * @param packState PackageState value
+   * @param option pointer to RteItem with options to pass
+   * @return a std::unique_ptr object holding an RteItemBuilder-derived pointer
   */
-  virtual std::unique_ptr<YmlTree> CreateUniqueYmlTree(IXmlItemBuilder* itemBuilder = nullptr) const;
+  virtual std::unique_ptr<RteItemBuilder> CreateUniqueRteItemBuilder(RteItem* rootParent = nullptr, PackageState packState = PackageState::PS_UNKNOWN,
+                                                                     const RteItem& options = RteItem::EMPTY_RTE_ITEM) const;
+
 
   /**
    * @brief save active project into cprj file

libs/rtemodel/include/RteTarget.h

@@ -117,6 +117,24 @@ class RteTarget : public RteItem
   */
   void SetTargetSupported(bool supported) { m_bTargetSupported = supported; }
 
+
+  /**
+   * @brief expands key sequences ("@L", "%L", etc.) or access sequences in the supplied string.
+   * @param str string to expand
+   * @param bUseAccessSequences expand access sequences instead of key sequences, default is false
+   * @param context pointer to RteItem representing expansion context (optional)
+   * @return expanded string
+  */
+  std::string ExpandString(const std::string& str,
+    bool bUseAccessSequences = false, RteItem* context = nullptr) const override;
+
+  /**
+   * @brief expand string by replacing $keyword$ with corresponding values
+   * @param src source string to expand
+   * @return expanded string
+  */
+  std::string ExpandAccessSequences(const std::string& src) const;
+
   /**
    * @brief return pointer to a filter object of type RteConditionContext
    * @return pointer to a filter object of type RteConditionContext

libs/rtemodel/src/RteGenerator.cpp

@@ -132,7 +132,7 @@ const string RteGenerator::GetCommand(const std::string& hostType) const
   return GetItemValue("command");
 }
 
-string RteGenerator::GetExecutable(const std::string& hostType) const
+string RteGenerator::GetExecutable(RteTarget* target, const std::string& hostType) const
 {
   string cmd = GetCommand(hostType);
   if (cmd.empty())
@@ -141,18 +141,29 @@ string RteGenerator::GetExecutable(const std::string& hostType) const
   if (cmd.find("http:") == 0 || cmd.find("https:") == 0) // a URL?
     return EMPTY_STRING; // return empty string here , GetExpandedWebLine() will return URL then
 
-  cmd = ExpandString(cmd);
+  if(target && IsExternal()) {
+    cmd = target->ExpandAccessSequences(cmd);
+  }else {
+    cmd = ExpandString(cmd);
+  }
 
   if (RteFsUtils::IsRelative(cmd)) {
-     cmd = GetAbsolutePackagePath() + cmd;
+     cmd = RteFsUtils::MakePathCanonical(GetAbsolutePackagePath() + cmd);
   }
   return cmd;
 }
 
 
-vector<pair<string, string> > RteGenerator::GetExpandedArguments(const string& hostType, bool dryRun) const
+vector<pair<string, string> > RteGenerator::GetExpandedArguments(RteTarget* target, const string& hostType, bool dryRun) const
 {
   vector<pair<string, string> > args;
+  if(IsExternal()) {
+  // add cbuild-gen-idx.yml
+    string idxFile = target->ExpandAccessSequences("$SolutionDir$/$Project$.$TargetType$.cbuild-gen-idx.yml");
+    args.push_back({RteUtils::EMPTY_STRING, idxFile});
+    return args;
+  }
+
   RteItem* argsItem = GetArgumentsItem("exe");
   if (argsItem) {
     for (auto arg : argsItem->GetChildren()) {
@@ -166,10 +177,10 @@ vector<pair<string, string> > RteGenerator::GetExpandedArguments(const string& h
   return args;
 }
 
-string RteGenerator::GetExpandedCommandLine(const string& hostType, bool dryRun) const
+string RteGenerator::GetExpandedCommandLine(RteTarget* target, const string& hostType, bool dryRun) const
 {
-  const vector<pair<string, string> > args = GetExpandedArguments(hostType, dryRun);
-  string fullCmd = RteUtils::AddQuotesIfSpace(GetExecutable(hostType));
+  string fullCmd = RteUtils::AddQuotesIfSpace(GetExecutable(target, hostType));
+  const vector<pair<string, string> > args = GetExpandedArguments(target, hostType, dryRun);
   for (size_t i = 0; i < args.size(); i++) {
     fullCmd += ' ' + RteUtils::AddQuotesIfSpace(args[i].first + args[i].second);
   }
@@ -215,11 +226,16 @@ string RteGenerator::GetExpandedWebLine(RteTarget* target) const
 
 string RteGenerator::GetExpandedGpdsc(RteTarget* target, const string& genDir) const
 {
-  string gpdsc = GetGpdsc();
-  if (gpdsc.empty()) {
-    gpdsc = target->GetProject()->GetName() + ".gpdsc";
+  string gpdsc;
+  if(IsExternal()) {
+    gpdsc = target->GetName() + ".cgen.yml";
   } else {
-    gpdsc = ExpandString(gpdsc);
+    gpdsc = GetGpdsc();
+    if(gpdsc.empty()) {
+      gpdsc = target->GetProject()->GetName() + ".gpdsc";
+    } else {
+      gpdsc = ExpandString(gpdsc);
+    }
   }
 
   if (!genDir.empty() && fs::path(gpdsc).is_absolute()) {
@@ -236,9 +252,9 @@ string RteGenerator::GetExpandedGpdsc(RteTarget* target, const string& genDir) c
 
 string RteGenerator::GetExpandedWorkingDir(RteTarget* target, const string& genDir) const
 {
-  string wd = genDir.empty() ? ExpandString(GetWorkingDir()) : genDir;
+  string wd = genDir.empty() ? ExpandString(GetWorkingDir(), IsExternal(), target) : genDir;
   fs::path path(wd);
-  if (wd.empty() || path.is_relative()) {
+  if (wd.empty() || (path.is_relative() && !IsExternal())) {
     // use project directory
     wd = target->GetProject()->GetProjectPath() + wd;
   }

libs/rtemodel/src/RteItem.cpp

@@ -821,11 +821,13 @@ string RteItem::GetOriginalAbsolutePath(const string& name) const
   return RteFsUtils::MakePathCanonical(absPath);
 }
 
-string RteItem::ExpandString(const string& str) const
+string RteItem::ExpandString(const string& str, bool bUseAccessSequences, RteItem* context) const
 {
   if (str.empty())
     return str;
-
+  if(context && context != this) {
+    return context->ExpandString(str, bUseAccessSequences, context);
+  }
   RteCallback* pCallback = GetCallback();
   if (pCallback)
     return pCallback->ExpandString(str);

libs/rtemodel/src/RteItemBuilder.cpp

@@ -32,7 +32,7 @@ RteItemBuilder::RteItemBuilder(RteItem* rootParent, PackageState packState) :
 RteItem* RteItemBuilder::CreateRootItem(const string& tag)
 {
   RteItem* pRoot = nullptr;
-  if (tag == "package") {
+  if (tag == "package" || tag == "generator-import") {
     RtePackage* pack = new RtePackage(m_rootParent, m_packState);
     m_packs.push_back(pack);
     pRoot = pack;

libs/rtemodel/src/RteKernel.cpp

@@ -210,14 +210,14 @@ CprjFile* RteKernel::ParseCprj(const string& cprjFile)
 {
   string msg = "Loading '" + cprjFile + "'";
   GetRteCallback()->OutputInfoMessage(msg);
-  RteItemBuilder rteItemBuilder;
-  unique_ptr<XMLTree> xmlTree = CreateUniqueXmlTree(&rteItemBuilder);
+  auto rteItemBuilder = CreateUniqueRteItemBuilder();
+  unique_ptr<XMLTree> xmlTree = CreateUniqueXmlTree(rteItemBuilder.get());
   if (!xmlTree->AddFileName(cprjFile, true)) {
     GetRteCallback()->Err("R811", R811, cprjFile);
     GetRteCallback()->OutputMessages(xmlTree->GetErrorStrings());
     return nullptr;
   }
-  CprjFile* cprj = rteItemBuilder.GetCprjFile();
+  CprjFile* cprj = rteItemBuilder->GetCprjFile();
 
   if (!cprj || !cprj->Validate()) {
     GetRteCallback()->Err("R812", R812, cprjFile);
@@ -254,14 +254,14 @@ void RteKernel::LoadExternalGenerators()
   list<string> files;
   RteFsUtils::GetMatchingFiles(files, ".generator.yml", etcDir, 1, true);
   RteGlobalModel* globalModel = GetGlobalModel();
-  RteItemBuilder rteBuilder(globalModel);
-  unique_ptr<YmlTree> ymlTree = CreateUniqueYmlTree(&rteBuilder);
+  auto rteItemBuilder = CreateUniqueRteItemBuilder(globalModel);
+  unique_ptr<XMLTree> ymlTree = CreateUniqueXmlTree(rteItemBuilder.get(), ".yml");
   for(auto& f : files) {
     if(contains_key(m_externalGeneratorFiles, f)) {
       continue;
     }
     bool result = ymlTree->ParseFile(f);
-    RteItem* rootItem = rteBuilder.GetRoot();
+    RteItem* rootItem = rteItemBuilder->GetRoot();
     if(result && rootItem) {
       m_externalGeneratorFiles[f] = rootItem;
       for(auto item : rootItem->GetChildren()) {
@@ -271,7 +271,7 @@ void RteKernel::LoadExternalGenerators()
         }
       }
     }
-    rteBuilder.Clear(false);
+    rteItemBuilder->Clear(false);
   }
 }
 
@@ -295,10 +295,10 @@ RtePackage* RteKernel::LoadPack(const string& pdscFile, PackageState packState)
     return pack;
   }
 
-  RteItemBuilder rteItemBuilder(GetGlobalModel(), packState);
-  unique_ptr<XMLTree> xmlTree = CreateUniqueXmlTree(&rteItemBuilder);
+  auto rteItemBuilder= CreateUniqueRteItemBuilder(GetGlobalModel(), packState);
+  unique_ptr<XMLTree> xmlTree = CreateUniqueXmlTree(rteItemBuilder.get());
   bool success = xmlTree->AddFileName(pdscFile, true);
-  pack = rteItemBuilder.GetPack();
+  pack = rteItemBuilder->GetPack();
   if (!success || !pack) {
     GetRteCallback()->Err("R802", R802, pdscFile);
     GetRteCallback()->OutputMessages(xmlTree->GetErrorStrings());
@@ -325,13 +325,13 @@ bool RteKernel::LoadPacks(const std::list<std::string>& pdscFiles, std::list<Rte
   RtePackRegistry* packRegistry = GetPackRegistry();
   unique_ptr<XMLTree> xmlTree = CreateUniqueXmlTree();
   for(auto& pdscFile : pdscFiles) {
-    RteItemBuilder rteItemBuilder(model, model->GetPackageState());
-    xmlTree->SetXmlItemBuilder(&rteItemBuilder);
+    auto rteItemBuilder = CreateUniqueRteItemBuilder(model, model->GetPackageState());
+    xmlTree->SetXmlItemBuilder(rteItemBuilder.get());
     if(bReplace) {
       packRegistry->ErasePack(pdscFile);
     }
     bool result = xmlTree->AddFileName(pdscFile, true);
-    RtePackage* pack = rteItemBuilder.GetPack();
+    RtePackage* pack = rteItemBuilder->GetPack();
     if(!result || !pack) {
       GetRteCallback()->Err("R802", R802, pdscFile);
       GetRteCallback()->OutputMessages(xmlTree->GetErrorStrings());
@@ -602,24 +602,25 @@ bool RteKernel::GetLocalPacksUrls(const string& rtePath, list<string>& urls) con
   return true;
 }
 
-unique_ptr<XMLTree> RteKernel::CreateUniqueXmlTree(IXmlItemBuilder* itemBuilder) const
+unique_ptr<XMLTree> RteKernel::CreateUniqueXmlTree(IXmlItemBuilder* itemBuilder, const std::string& ext) const
 {
-  unique_ptr<XMLTree> xmlTree(CreateXmlTree(itemBuilder));
+  bool bYaml = !ext.empty() && (ext == ".yml" || ext == ".yaml");
+  XMLTree* pXmlTree = bYaml ? CreateYmlTree(itemBuilder) : CreateXmlTree(itemBuilder);
+  unique_ptr<XMLTree> xmlTree(pXmlTree);
   if (xmlTree.get() != nullptr) {
     xmlTree->SetCallback(GetRteCallback());
     xmlTree->Init();
   }
   return xmlTree;
 }
 
-unique_ptr<YmlTree> RteKernel::CreateUniqueYmlTree(IXmlItemBuilder* itemBuilder) const
+unique_ptr<RteItemBuilder> RteKernel::CreateUniqueRteItemBuilder(RteItem* rootParent, PackageState packState, const RteItem& options) const
 {
-  unique_ptr<YmlTree> ymlTree(CreateYmlTree(itemBuilder));
-  if (ymlTree.get() != nullptr) {
-    ymlTree->SetCallback(GetRteCallback());
-    ymlTree->Init();
+  unique_ptr<RteItemBuilder> builder( new RteItemBuilder(rootParent, packState));
+  if(!options.IsEmpty()) {
+    builder->SetAttributes(options);
   }
-  return ymlTree;
+  return builder;
 }
 
 YmlTree* RteKernel::CreateYmlTree(IXmlItemBuilder* itemBuilder) const