[projmgr] External Generator PoC - Initial commit

Author: brondani

tools/projmgr/CMakeLists.txt

@@ -13,12 +13,12 @@ set_property(DIRECTORY PROPERTY VS_STARTUP_PROJECT projmgr)
 SET(PROJMGR_SOURCE_FILES ProjMgr.cpp ProjMgrKernel.cpp ProjMgrCallback.cpp
   ProjMgrParser.cpp ProjMgrWorker.cpp ProjMgrGenerator.cpp ProjMgrXmlParser.cpp
   ProjMgrYamlParser.cpp ProjMgrLogger.cpp ProjMgrYamlSchemaChecker.cpp
-  ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp
+  ProjMgrYamlEmitter.cpp ProjMgrUtils.cpp ProjMgrExtGenerator.cpp
 )
 SET(PROJMGR_HEADER_FILES ProjMgr.h ProjMgrKernel.h ProjMgrCallback.h
   ProjMgrParser.h ProjMgrWorker.h ProjMgrGenerator.h ProjMgrXmlParser.h
   ProjMgrYamlParser.h ProjMgrLogger.h ProjMgrYamlSchemaChecker.h
-  ProjMgrYamlEmitter.h ProjMgrUtils.h
+  ProjMgrYamlEmitter.h ProjMgrUtils.h ProjMgrExtGenerator.h
 )
 
 list(TRANSFORM PROJMGR_SOURCE_FILES PREPEND src/)

tools/projmgr/include/ProjMgr.h

@@ -88,6 +88,7 @@ class ProjMgr {
   ProjMgrWorker m_worker;
   ProjMgrGenerator m_generator;
   ProjMgrYamlEmitter m_emitter;
+  ProjMgrExtGenerator m_extGenerator;
 
   std::string m_csolutionFile;
   std::string m_cdefaultFile;

tools/projmgr/include/ProjMgrExtGenerator.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2020-2023 Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef PROJMGREXTGENERATOR_H
+#define PROJMGREXTGENERATOR_H
+
+#include "ProjMgrParser.h"
+#include "ProjMgrUtils.h"
+
+ /**
+  * @brief external generator item containing
+  *        component identifier
+  *        directory for generated files
+  *        project type
+ */
+struct ExtGeneratorItem {
+  std::string componentId;
+  std::string genDir;
+  std::string projectType;
+};
+
+
+struct CbuildGenItem {
+  StrVec forContext;
+  std::string device;
+  std::string board;
+  std::string projectPart;
+};
+
+/**
+ * @brief map of used generators, directories and contexts
+*/
+typedef std::map<std::string, StrVecMap> GeneratorContextVecMap;
+
+/**
+ * @brief vector of external generator items
+*/
+typedef std::vector<ExtGeneratorItem> ExtGeneratorVec;
+
+/**
+ * @brief map of vector of external generator items
+*/
+typedef std::map<std::string, ExtGeneratorVec> ExtGeneratorVecMap;
+
+/**
+ * @brief solution/project types
+*/
+static constexpr const char* TYPE_SINGLE_CORE = "single-core";
+static constexpr const char* TYPE_MULTI_CORE = "multi-core";
+static constexpr const char* TYPE_TRUSTZONE = "trustzone";
+static constexpr const char* PROCESSOR_CORE = "processor-core";
+
+/**
+ * @brief projmgr external generator class responsible for handling global generators
+*/
+class ProjMgrExtGenerator {
+public:
+  /**
+   * @brief class constructor
+  */
+  ProjMgrExtGenerator(ProjMgrParser* parser);
+
+
+  /**
+   * @brief class destructor
+  */
+  ~ProjMgrExtGenerator(void);
+
+
+  bool RetrieveGlobalGenerators(void);
+
+  bool IsGlobalGenerator(const std::string& generatorId);
+  bool CheckGeneratorId(const std::string& generatorId, const std::string& componentId);
+  const std::string& GetGlobalGenDir(const std::string& generatorId);
+  const std::string& GetGlobalGenRunCmd(const std::string& generatorId);
+  void AddUsedGenerator(const std::string& generatorId, const std::string& genDir, const std::string& contextId);
+  const GeneratorContextVecMap& GetUsedGenerators(void);
+  bool GetCgen(const std::string& contextId, ClayerItem& cgen);
+
+protected:
+  ProjMgrParser* m_parser = nullptr;
+  StrVec m_globalGeneratorFiles;
+  std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
+  GeneratorContextVecMap m_usedGenerators;
+  
+  bool m_checkSchema;
+  std::string m_compilerRoot;
+
+
+
+  
+};
+
+#endif  // PROJMGREXTGENERATOR_H

tools/projmgr/include/ProjMgrParser.h

@@ -140,12 +140,14 @@ struct TargetType {
  * @brief directories item containing
  *        intdir directory,
  *        outdir directory,
+ *        cbuild directory,
  *        cprj directory,
  *        rte directory,
 */
 struct DirectoriesItem {
   std::string intdir;
   std::string outdir;
+  std::string cbuild;
   std::string cprj;
   std::string rte;
 };
@@ -402,6 +404,20 @@ struct ClayerItem {
   GeneratorsItem generators;
 };
 
+/**
+ * @brief global generator item containing
+ *        generator id,
+ *        download url,
+ *        bridge program,
+ *        output
+*/
+struct GlobalGeneratorItem {
+  std::string id;
+  std::string downloadUrl;
+  std::string run;
+  std::string output;
+};
+
 /**
  * @brief projmgr parser class for public interfacing
 */
@@ -448,6 +464,12 @@ class ProjMgrParser {
   */
   bool ParseGenericClayer(const std::string& input, bool checkSchema);
 
+  /**
+   * @brief parse global generator
+   * @param input generator.yml file
+  */
+  bool ParseGlobalGenerator(const std::string& input, bool checkSchema);
+
   /**
    * @brief get cdefault
    * @return cdefault item
@@ -478,12 +500,19 @@ class ProjMgrParser {
   */
   std::map<std::string, ClayerItem>& GetGenericClayers(void);
 
+  /**
+   * @brief get global generators
+   * @return global generators map
+  */
+  std::map<std::string, GlobalGeneratorItem>& GetGlobalGenerators(void);
+
 protected:
   CdefaultItem m_cdefault;
   CsolutionItem m_csolution;
   std::map<std::string, CprojectItem> m_cprojects;
   std::map<std::string, ClayerItem> m_clayers;
   std::map<std::string, ClayerItem> m_genericClayers;
+  std::map<std::string, GlobalGeneratorItem> m_globalGenerators;
 };
 
 #endif  // PROJMGRPARSER_H

tools/projmgr/include/ProjMgrWorker.h

@@ -7,6 +7,7 @@
 #ifndef PROJMGRWORKER_H
 #define PROJMGRWORKER_H
 
+#include "ProjMgrExtGenerator.h"
 #include "ProjMgrKernel.h"
 #include "ProjMgrParser.h"
 #include "ProjMgrUtils.h"
@@ -252,6 +253,7 @@ struct ContextTypesItem {
  *        output type,
  *        device selection,
  *        board selection,
+ *        device item struct,
  *        list of package requirements,
  *        map of required pdsc files and optionally its local path
  *        list of component requirements,
@@ -269,6 +271,7 @@ struct ContextTypesItem {
  *        valid connections,
  *        linker options,
  *        map of variables,
+ *        external generator directory,
  *        boolean processed precedences
 */
 struct ContextItem {
@@ -289,6 +292,7 @@ struct ContextItem {
   OutputTypes outputTypes;
   std::string device;
   std::string board;
+  DeviceItem deviceItem;
   std::vector<PackageItem> packRequirements;
   std::map<std::string, std::pair<std::string, std::string>> pdscFiles;
   std::vector<PackInfo>missingPacks;
@@ -310,6 +314,7 @@ struct ContextItem {
   std::vector<ConnectionsCollectionVec> validConnections;
   LinkerContextItem linker;
   std::map<std::string, std::string> variables;
+  StrMap extGenDir;
   bool precedences;
 };
 
@@ -362,28 +367,22 @@ class ProjMgrWorker {
   /**
    * @brief class constructor
   */
-  ProjMgrWorker(void);
+  ProjMgrWorker(ProjMgrParser* parser, ProjMgrExtGenerator* extGenerator);
 
   /**
    * @brief class destructor
   */
   ~ProjMgrWorker(void);
 
-  /**
-   * @brief set parser
-   * @param pointer to parser
-  */
-  void SetParser(ProjMgrParser* parser);
-
   /**
    * @brief process context
    * @param reference to context
-   * @param loadGpdsc boolean automatically load gpdsc, default true
+   * @param loadGenFiles boolean automatically load generated files, default true
    * @param resolveDependencies boolean automatically resolve dependencies, default true
    * @param updateRteFiles boolean update RTE files, default true
    * @return true if executed successfully
   */
-  bool ProcessContext(ContextItem& context, bool loadGpdsc = true, bool resolveDependencies = true, bool updateRteFiles = true);
+  bool ProcessContext(ContextItem& context, bool loadGenFiles = true, bool resolveDependencies = true, bool updateRteFiles = true);
 
   /**
    * @brief list available packs
@@ -554,6 +553,8 @@ class ProjMgrWorker {
   */
   bool ExecuteGenerator(std::string& generatorId);
 
+  bool ExecuteExtGenerator(std::string& generatorId);
+
   /**
    * @brief initialize model
    * @return true if executed successfully
@@ -605,10 +606,14 @@ class ProjMgrWorker {
   */
   void PrintMissingFilters(void);
 
+  bool ProcessGlobalGenerators(ContextItem* selectedContext, const std::string& generatorId,
+    std::string& projectType, StrVec& siblings);
+
 protected:
   ProjMgrParser* m_parser = nullptr;
   ProjMgrKernel* m_kernel = nullptr;
   RteGlobalModel* m_model = nullptr;
+  ProjMgrExtGenerator* m_extGenerator = nullptr;
   std::list<RtePackage*> m_loadedPacks;
   std::vector<ToolchainItem> m_toolchains;
   StrVec m_toolchainConfigFiles;
@@ -640,7 +645,7 @@ class ProjMgrWorker {
   bool GetProjectSetup(ContextItem& context);
   bool InitializeTarget(ContextItem& context);
   bool SetTargetAttributes(ContextItem& context, std::map<std::string, std::string>& attributes);
-  bool ProcessPrecedences(ContextItem& context);
+  bool ProcessPrecedences(ContextItem& context, bool rerun = false);
   bool ProcessPrecedence(StringCollection& item);
   bool ProcessCompilerPrecedence(StringCollection& item, bool acceptRedefinition = false);
   bool ProcessDevice(ContextItem& context);
@@ -722,10 +727,13 @@ class ProjMgrWorker {
   void UpdatePartialReferencedContext(ContextItem& context, std::string& contextName);
   void ExpandAccessSequence(const ContextItem& context, const ContextItem& refContext, const std::string& sequence, std::string& item, bool withHeadingDot);
   bool GetGeneratorDir(const RteGenerator* generator, ContextItem& context, const std::string& layer, std::string& genDir);
+  bool GetGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
+  bool GetExtGeneratorDir(const std::string& generatorId, ContextItem& context, const std::string& layer, std::string& genDir);
   bool ParseContextLayers(ContextItem& context);
   bool AddPackRequirements(ContextItem& context, const std::vector<PackItem> packRequirements);
   void CheckTypeFilterSpelling(const TypeFilter& typeFilter);
   void CheckCompilerFilterSpelling(const std::string& compiler);
+  bool ProcessGeneratedLayers(ContextItem& context);
 };
 
 #endif  // PROJMGRWORKER_H

tools/projmgr/include/ProjMgrYamlEmitter.h

@@ -33,12 +33,17 @@ class ProjMgrYamlEmitter {
   */
   static bool GenerateCbuildIndex(ProjMgrParser& parser, const std::vector<ContextItem*> contexts, const std::string& outputDir);
 
+  static bool GenerateCbuildGenIndex(ProjMgrParser& parser, const std::vector<ContextItem*> siblings,
+    const std::string& type, const std::string& outputDir);
+
   /**
    * @brief generate cbuild.yml file
    * @param context pointer to the context
    * @return true if executed successfully
   */
   static bool GenerateCbuild(ContextItem* context);
+
+  static bool GenerateCbuildGen(ContextItem* context, const std::string& output);
 };
 
 #endif  // PROJMGRYAMLEMITTER_H

tools/projmgr/include/ProjMgrYamlParser.h

@@ -21,11 +21,15 @@ static constexpr const char* YAML_BASE_DIR = "base-dir";
 static constexpr const char* YAML_BASE_NAME = "base-name";
 static constexpr const char* YAML_BOARD = "board";
 static constexpr const char* YAML_BUILD = "build";
+static constexpr const char* YAML_BUILD_GEN = "build-gen";
 static constexpr const char* YAML_BUILD_IDX = "build-idx";
+static constexpr const char* YAML_BUILD_GEN_IDX = "build-gen-idx";
 static constexpr const char* YAML_BUILDTYPES = "build-types";
 static constexpr const char* YAML_CATEGORY = "category";
 static constexpr const char* YAML_CBUILDS = "cbuilds";
 static constexpr const char* YAML_CBUILD = "cbuild";
+static constexpr const char* YAML_CBUILD_GENS = "cbuild-gens";
+static constexpr const char* YAML_CBUILD_GEN = "cbuild-gen";
 static constexpr const char* YAML_CDEFAULT = "cdefault";
 static constexpr const char* YAML_CLAYERS = "clayers";
 static constexpr const char* YAML_CLAYER = "clayer";
@@ -53,6 +57,7 @@ static constexpr const char* YAML_DEFINE = "define";
 static constexpr const char* YAML_DELPATH = "del-path";
 static constexpr const char* YAML_DESCRIPTION = "description";
 static constexpr const char* YAML_DEVICE = "device";
+static constexpr const char* YAML_DOWNLOAD_URL = "download-url";
 static constexpr const char* YAML_ENDIAN = "endian";
 static constexpr const char* YAML_FILE = "file";
 static constexpr const char* YAML_FILES = "files";
@@ -61,6 +66,7 @@ static constexpr const char* YAML_FORBOARD = "for-board";
 static constexpr const char* YAML_FORCOMPILER = "for-compiler";
 static constexpr const char* YAML_FORCONTEXT = "for-context";
 static constexpr const char* YAML_FORDEVICE = "for-device";
+static constexpr const char* YAML_FORPROJECTPART = "for-project-part";
 static constexpr const char* YAML_FPU = "fpu";
 static constexpr const char* YAML_GENERATED_BY = "generated-by";
 static constexpr const char* YAML_GENERATOR = "generator";
@@ -106,16 +112,19 @@ static constexpr const char* YAML_PATH = "path";
 static constexpr const char* YAML_PROCESSOR = "processor";
 static constexpr const char* YAML_PROJECT = "project";
 static constexpr const char* YAML_PROJECTS = "projects";
+static constexpr const char* YAML_PROJECT_TYPE = "project-type";
 static constexpr const char* YAML_PROVIDES = "provides";
 static constexpr const char* YAML_REGIONS = "regions";
 static constexpr const char* YAML_RTE = "rte";
+static constexpr const char* YAML_RUN = "run";
 static constexpr const char* YAML_SCOPE = "scope";
 static constexpr const char* YAML_SCRIPT = "script";
 static constexpr const char* YAML_SOLUTION = "solution";
 static constexpr const char* YAML_SELECTED_BY = "selected-by";
 static constexpr const char* YAML_SETUPS = "setups";
 static constexpr const char* YAML_SETUP = "setup";
 static constexpr const char* YAML_SET = "set";
+static constexpr const char* YAML_SUB_DIR = "sub-dir";
 static constexpr const char* YAML_SWITCH = "switch";
 static constexpr const char* YAML_TARGETTYPES = "target-types";
 static constexpr const char* YAML_TRUSTZONE = "trustzone";
@@ -177,6 +186,15 @@ class ProjMgrYamlParser {
   bool ParseClayer(const std::string& input, std::map<std::string,
     ClayerItem>& clayers, bool checkSchema);
 
+  /**
+   * @brief parse global generator
+   * @param input generator.yml file
+   * @param reference to store parsed generator item
+   * @param checkSchema false to skip schema validation
+  */
+  bool ParseGlobalGenerator(const std::string& input,
+    std::map<std::string, GlobalGeneratorItem>& generators, bool checkSchema);
+
 protected:
   void ParseMisc(const YAML::Node& parent, std::vector<MiscItem>& misc);
   void ParseDefine(const YAML::Node& parent, std::vector<std::string>& define);