diff --git a/vcpkg-configuration.json b/.ci/vcpkg-configuration.json similarity index 60% rename from vcpkg-configuration.json rename to .ci/vcpkg-configuration.json index c3b0064..d11f445 100644 --- a/vcpkg-configuration.json +++ b/.ci/vcpkg-configuration.json @@ -8,10 +8,9 @@ ], "requires": { "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.10.0", - "arm:compilers/arm/armclang": "^6.24.0", - "arm:models/arm/avh-fvp": "^11.29.27", + "arm:compilers/arm/armclang": "^6.23.0", + "arm:models/arm/avh-fvp": "^11.28.32", "arm:tools/kitware/cmake": "^3.31.5", - "arm:tools/ninja-build/ninja": "^1.12.1", - "arm:debuggers/arm/armdbg": "^6.5.0" + "arm:tools/ninja-build/ninja": "^1.12.0" } } \ No newline at end of file diff --git a/.github/workflows/test_audio.yaml b/.github/workflows/test_audio.yaml index b3b4064..a750340 100644 --- a/.github/workflows/test_audio.yaml +++ b/.github/workflows/test_audio.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: mlek audio build and execution test +name: MLEK audio build and execution test on: workflow_dispatch: @@ -120,7 +120,7 @@ jobs: - name: Install tools uses: ARM-software/cmsis-actions/vcpkg@v1 with: - config: "./ARM/cmsis-mlek/vcpkg-configuration.json" + config: ".ci/vcpkg-configuration.json" - name: Activate Arm tool license uses: ARM-software/cmsis-actions/armlm@v1 diff --git a/.github/workflows/test_generic.yaml b/.github/workflows/test_generic.yaml index 6cd9f2b..8295732 100644 --- a/.github/workflows/test_generic.yaml +++ b/.github/workflows/test_generic.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: mlek generic build and execution test +name: MLEK generic build and execution test on: workflow_dispatch: @@ -98,7 +98,7 @@ jobs: - name: Install tools uses: ARM-software/cmsis-actions/vcpkg@v1 with: - config: "vcpkg-configuration.json" + config: ".ci/vcpkg-configuration.json" - name: Activate Arm tool license uses: ARM-software/cmsis-actions/armlm@v1 diff --git a/.github/workflows/test_video.yaml b/.github/workflows/test_video.yaml index 8c7432c..c2230a8 100644 --- a/.github/workflows/test_video.yaml +++ b/.github/workflows/test_video.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: mlek video build and execution test +name: MLEK video build and execution test on: workflow_dispatch: @@ -93,7 +93,7 @@ jobs: - name: Install tools uses: ARM-software/cmsis-actions/vcpkg@v1 with: - config: "vcpkg-configuration.json" + config: ".ci/vcpkg-configuration.json" - name: Activate Arm tool license uses: ARM-software/cmsis-actions/armlm@v1 diff --git a/.gitignore b/.gitignore index ad90095..96f25c7 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ dkms.conf .DS_Store examples/audio/kws/.clangd template/audio/mlek_audio.cbuild-set.yml + +# Clangd +.clangd diff --git a/.vscode/settings.json b/.vscode/settings.json index 96db5c6..2dd0946 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { "clangd.arguments": [ - "--compile-commands-dir=/Users/chrsei01/02_Git/Arm-Examples/cmsis-mlek/template/audio/out/kws/${Name}/Debug-Live_Stream" + "--compile-commands-dir=c:\\w\\cmsis-mlek\\template\\audio\\out\\kws\\${Name}\\Debug-Live_Stream" ] } \ No newline at end of file diff --git a/README.md b/README.md index f109f83..c11c0d3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ # CMSIS-MLEK Software Pack -This repository contains **Machine Learning Evaluation Kit (MLEK)** pack which provides [CMSIS Reference Applications](https://open-cmsis-pack.github.io/cmsis-toolbox/ReferenceApplications/) and templates for Edge AI development. These applications implement data preprocessing, memory management, and neural network inference pipelines that are optimized for Cortex-M and Ethos-U platforms. +This repository contains **Machine Learning Evaluation Kit (MLEK)** pack which provides [CMSIS Reference Applications](https://open-cmsis-pack.github.io/cmsis-toolbox/ReferenceApplications/) and templates for Edge AI development. These applications implement data preprocessing, memory management, and neural network inference pipelines that are optimized for Cortex-M and Ethos-U platforms. The examples are prepared to run with [Keil Studio for VS Code](https://www.keil.arm.com/). + +The CMSIS-MLEK software pack is derived from the [Arm® ML embedded evaluation kit](https://git.gitlab.arm.com/artificial-intelligence/ethos-u/ml-embedded-evaluation-kit) and makes the examples easier to access. It also contains interfaces to physical hardware and simplifies porting to target hardware. It contains the following ML applications and uses currently Neural Network Models currently in [TensorFlow Lite](https://www.keil.arm.com/packs/tensorflow-lite-micro-tensorflow) format. + +ML application | Description | Neural Network Model +:----------------------------------------------|:------------------------|:--------------------- +[Keyword spotting (KWS)](./template/audio) | Recognize the presence of a key word in verbal speech | [MicroNet](https://github.com/ARM-software/ML-zoo/tree/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8) +[Object detection](./template/video) | Detects and draws face bounding box in a given image | [Yolo Fastest](https://github.com/emza-vs/ModelZoo/blob/master/object_detection/yolo-fastest_192_face_v4.tflite) +[Generic inference runner](./template/generic) | Code block allowing you to develop your own use case | Your custom model Refer to [Overview](./overview/README.md) for more details. @@ -9,7 +17,7 @@ Refer to [Overview](./overview/README.md) for more details. Directory | Description ----------------------------------|------------------------------- [overview](./overview) | Top-level overview of MLEK reference applications. -[documentation](./documentation/) | [User documentation](https://arm-examples.github.io/cmsis-mlek/main/index.html) of the MLEK reference applications. +[docs](./docs/) | Source of the [user documentation](https://arm-examples.github.io/cmsis-mlek) of the MLEK reference applications. [template](./template) | [MLEK reference applications](https://arm-examples.github.io/cmsis-mlek/main/template.html), a test framework for DSP and ML algorithms. [.github/workflows](./.github/workflows) | GitHub Actions for validation and publishing. [.ci](./.ci) | Files that relate to CI tests. @@ -17,6 +25,7 @@ Directory | Description ## Related - [MLEK Pack](https://www.keil.arm.com/packs/MLEK) available on [keil.arm.com/packs](https://www.keil.arm.com/packs). +- [tensorflow-lite-micro Pack](https://www.keil.arm.com/packs/tensorflow-lite-micro-tensorflow) available on [keil.arm.com/packs](https://www.keil.arm.com/packs). - [MLEK Examples](https://github.com/Arm-Examples/mlek-examples) a repository with MLEK applications that are configured for evaluation boards and use cases. - [ML Developers Guide for Cortex-M Processors and Ethos-U NPU](https://developer.arm.com/documentation/109267). - [Arm Virtual Hardware - FVP](https://github.com/arm-software/avh) repository with documentation. @@ -27,11 +36,7 @@ The MLEK references applications are licensed under [![License](https://img.shie ## Documentation -The [documentation](https://arm-examples.github.io/cmsis-mlek/index.html) is generated using [MKDocs](https://www.mkdocs.org/) with the following additional plugins: - -- [mermaid2](https://mkdocs-mermaid2.readthedocs.io/en/latest/) for sequence diagrams. - -Use `mkdocs serve` to generate the documentation on a local computer. +The [documentation](https://arm-examples.github.io/cmsis-mlek/index.html) is generated using [MKDocs](https://www.mkdocs.org/). Use `mkdocs serve` to generate the documentation on a local computer. ## Contributions and Pull Requests diff --git a/docs/overview.md b/docs/overview.md index 1a3ce56..3c5f7db 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -10,14 +10,15 @@ Key Features: - **Performance Validation**: Test and optimize ML performance on target hardware or simulation. - **Hardware Evaluation**: Compare performance across different Corstone platforms and configurations. -MLEK covers three main application domains: +The CMSIS-MLEK software pack is derived from the [Arm® ML embedded evaluation kit](https://git.gitlab.arm.com/artificial-intelligence/ethos-u/ml-embedded-evaluation-kit) and makes the examples easier to access. It also contains interfaces to physical hardware and simplifies porting to target hardware. It contains the following ML applications and uses currently Neural Network Models currently in [TensorFlow Lite](https://www.keil.arm.com/packs/tensorflow-lite-micro-tensorflow) format. -- [Audio](https://arm-examples.github.io/cmsis-mlek/templates_audio.html) processing with audio classification and keyword spotting. -- [Video](https://arm-examples.github.io/cmsis-mlek/templates_video.html) processing with image classification and object detection. -- [Generic](https://arm-examples.github.io/cmsis-mlek) inference for flexible ML model deployment. +| ML application | Description | Neural Network Model | +|:-----------------------------------------------|:------------------------|:----------------------| +| [Keyword spotting (KWS)](https://github.com/ARM-software/cmsis-mlek/template/audio) | Recognize the presence of a key word in verbal speech | [MicroNet](https://github.com/ARM-software/ML-zoo/tree/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8) | +| [Object detection](https://github.com/ARM-software/cmsis-mlek/template/video) | Detects and draws face bounding box in a given image | [Yolo Fastest](https://github.com/emza-vs/ModelZoo/blob/master/object_detection/yolo-fastest_192_face_v4.tflite) | +| [Generic inference runner](https://github.com/ARM-software/cmsis-mlek/template/generic) | Code block allowing you to develop your own use case | Your custom model | - -Each reference applications is a *csolution project* which supports deployment to physical hardware or Arm Virtual Hardware (AVH-FVP) for simulation. +Each [ML reference application](https://open-cmsis-pack.github.io/cmsis-toolbox/ReferenceApplications/) is a *csolution project* which supports deployment to physical hardware or Arm Virtual Hardware (AVH-FVP) for simulation. ![MLEK Reference Application Architecture](./images/MLEK-Architecture.png) @@ -27,24 +28,26 @@ A board layer (`*.clayer.yml`) implements the drivers for the physical interface |:----------------------------|:----------------| | **Audio Processing** | | | CMSIS_VSTREAM_AUDIO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Audio input. | +| STDOUT | Standard I/O for printf output. | | **Video Processing** | | | CMSIS_VSTREAM_VIDEO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Video input. | | CMSIS_VSTREAM_VIDEO_OUT | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Video output. | +| STDOUT | Standard I/O for printf output. | | **Generic Inference Runner**| | -| CMSIS_VSTREAM_AUDIO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html configured for Audio input. | +| STDOUT | Standard I/O for printf output. | ## Platform Support -The templates support via _target names_ multiple [Arm Cortex-M IP Subsystems](https://www.arm.com/products/silicon-ip-subsystems#Products). +The templates support via _target names_ multiple [Arm Cortex-M IP Subsystems](https://www.arm.com/products/silicon-ip-subsystems#Products). These _target names_ support execution on [AVH FVP simulation models](https://arm-software.github.io/AVH/main/simulation/html/index.html) which is useful during software development or with [Contiguous Integration (CI)](https://github.com/Arm-Examples/cmsis-mlek/actions) testing using GitHub actions. -| IP Subsystem | Description | -|:-------------|:-------------| -| Corstone-300 | Cortex-M55 optional with Ethos-U55 or Ethos-U65 | -| Corstone-310 | Cortex-M85 optional with Ethos-U55 | -| Corstone-315 | Cortex-M85 optional with Ethos-U65 | -| Corstone-320 | Cortex-M85 optional with Ethos-U85 | +| Target Name | IP Subsystem | Description | +|:-------------|:-------------|:-------------| +| AVH-SSE-300 | [Corstone-300](https://developer.arm.com/Processors/Corstone-300) | Cortex-M55 optional with Ethos-U55 or Ethos-U65 | +| AVH-SSE-310 | [Corstone-310](https://developer.arm.com/Processors/Corstone-310) | Cortex-M85 optional with Ethos-U55 | +| AVH-SSE-315 | [Corstone-315](https://developer.arm.com/Processors/Corstone-315) | Cortex-M85 optional with Ethos-U65 | +| AVH-SSE-320 | [Corstone-320](https://developer.arm.com/Processors/Corstone-320) | Cortex-M85 optional with Ethos-U85 | -Adding a postfix to the _target name_ in the `*.csolution.yml` project file configures the neural network inference pipeline for Ethos-U. Without this prefix only the Cortex-M system is used as shown in the diagram below. +Adding a postfix to the _target name_ in the `*.csolution.yml` project file configures the neural network inference pipeline for Ethos-U. Without this prefix only the Cortex-M system is used as shown in the diagram below. Such a postfix can also be used for _target names_ that deploy to physical hardware. | Postfix | Description | |:-------------|:------------| diff --git a/docs/template.md b/docs/template.md deleted file mode 100644 index e69de29..0000000 diff --git a/mkdocs.yml b/mkdocs.yml index ff1c15d..bd0103e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,7 +7,7 @@ nav: - Audio Template Applications: templates_audio.md - Video Template Applications: templates_video.md - Target Configuration Hardware Boards: target_configuration_refapp.md - - Target Configuration Arm Virtual Hardware: target_configuration_avh.md + - Target Configuration Arm Virtual Hardware: target_configuration_avh.md plugins: - search diff --git a/overview/README.md b/overview/README.md index 2a33fe3..eec8b93 100644 --- a/overview/README.md +++ b/overview/README.md @@ -10,13 +10,15 @@ Key Features: - **Performance Validation**: Test and optimize ML performance on target hardware or simulation. - **Hardware Evaluation**: Compare performance across different Corstone platforms and configurations. -MLEK covers three main application domains: +The CMSIS-MLEK software pack is derived from the [Arm® ML embedded evaluation kit](https://git.gitlab.arm.com/artificial-intelligence/ethos-u/ml-embedded-evaluation-kit) and makes the examples easier to access. It also contains interfaces to physical hardware and simplifies porting to target hardware. It contains the following ML applications and uses currently Neural Network Models currently in [TensorFlow Lite](https://www.keil.arm.com/packs/tensorflow-lite-micro-tensorflow) format. -- [Audio](https://arm-examples.github.io/cmsis-mlek/templates_audio.html) processing with audio classification and keyword spotting. -- [Video](https://arm-examples.github.io/cmsis-mlek/templates_video.html) processing with image classification and object detection. -- [Generic](https://arm-examples.github.io/cmsis-mlek) inference for flexible ML model deployment. +| ML application | Description | Neural Network Model | +|:-----------------------------------------------|:------------------------|:----------------------| +| [Keyword spotting (KWS)](https://github.com/ARM-software/cmsis-mlek/template/audio) | Recognize the presence of a key word in verbal speech | [MicroNet](https://github.com/ARM-software/ML-zoo/tree/9f506fe52b39df545f0e6c5ff9223f671bc5ae00/models/keyword_spotting/micronet_medium/tflite_int8) | +| [Object detection](https://github.com/ARM-software/cmsis-mlek/template/video) | Detects and draws face bounding box in a given image | [Yolo Fastest](https://github.com/emza-vs/ModelZoo/blob/master/object_detection/yolo-fastest_192_face_v4.tflite) | +| [Generic inference runner](https://github.com/ARM-software/cmsis-mlek/template/generic) | Code block allowing you to develop your own use case | Your custom model | -Each reference applications is a *csolution project* which supports deployment to physical hardware or Arm Virtual Hardware (AVH-FVP) for simulation. +Each [ML reference application](https://open-cmsis-pack.github.io/cmsis-toolbox/ReferenceApplications/) is a *csolution project* which supports deployment to physical hardware or Arm Virtual Hardware (AVH-FVP) for simulation. ![MLEK Reference Application Architecture](./MLEK-Architecture.png) @@ -26,24 +28,26 @@ A board layer (`*.clayer.yml`) implements the drivers for the physical interface |:----------------------------|:----------------| | **Audio Processing** | | | CMSIS_VSTREAM_AUDIO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Audio input. | +| STDOUT | Standard I/O for printf output. | | **Video Processing** | | | CMSIS_VSTREAM_VIDEO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Video input. | | CMSIS_VSTREAM_VIDEO_OUT | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html) configured for Video output. | +| STDOUT | Standard I/O for printf output. | | **Generic Inference Runner**| | -| CMSIS_VSTREAM_AUDIO_IN | [CMSIS-Driver vStream](https://arm-software.github.io/CMSIS_6/latest/Driver/group__mci__interface__gr.html configured for Audio input. | +| STDOUT | Standard I/O for printf output. | ## Platform Support -The templates support via _target names_ multiple [Arm Cortex-M IP Subsystems](https://www.arm.com/products/silicon-ip-subsystems#Products). +The templates support via _target names_ multiple [Arm Cortex-M IP Subsystems](https://www.arm.com/products/silicon-ip-subsystems#Products). These _target names_ support execution on [AVH FVP simulation models](https://arm-software.github.io/AVH/main/simulation/html/index.html) which is useful during software development or with [Contiguous Integration (CI)](https://github.com/Arm-Examples/cmsis-mlek/actions) testing using GitHub actions. -| IP Subsystem | Description | -|:-------------|:-------------| -| Corstone-300 | Cortex-M55 optional with Ethos-U55 or Ethos-U65 | -| Corstone-310 | Cortex-M85 optional with Ethos-U55 | -| Corstone-315 | Cortex-M85 optional with Ethos-U65 | -| Corstone-320 | Cortex-M85 optional with Ethos-U85 | +| Target Name | IP Subsystem | Description | +|:-------------|:-------------|:-------------| +| AVH-SSE-300 | [Corstone-300](https://developer.arm.com/Processors/Corstone-300) | Cortex-M55 optional with Ethos-U55 or Ethos-U65 | +| AVH-SSE-310 | [Corstone-310](https://developer.arm.com/Processors/Corstone-310) | Cortex-M85 optional with Ethos-U55 | +| AVH-SSE-315 | [Corstone-315](https://developer.arm.com/Processors/Corstone-315) | Cortex-M85 optional with Ethos-U65 | +| AVH-SSE-320 | [Corstone-320](https://developer.arm.com/Processors/Corstone-320) | Cortex-M85 optional with Ethos-U85 | -Adding a postfix to the _target name_ in the `*.csolution.yml` project file configures the neural network inference pipeline for Ethos-U. Without this prefix only the Cortex-M system is used as shown in the diagram below. +Adding a postfix to the _target name_ in the `*.csolution.yml` project file configures the neural network inference pipeline for Ethos-U. Without this prefix only the Cortex-M system is used as shown in the diagram below. Such a postfix can also be used for _target names_ that deploy to physical hardware. | Postfix | Description | |:-------------|:------------| @@ -56,8 +60,6 @@ Adding a postfix to the _target name_ in the `*.csolution.yml` project file conf ## Links -- [Documentation](../documentation/index.md) -- [Audio Templates](../template/audio/) -- [Video Templates](../template/video/) -- [Generic Templates](../template/generic/) -- [CMSIS-MLEK Repository](https://github.com/ARM-software/cmsis-mlek) +- [Documentation](https://arm-examples.github.io/cmsis-mlek) +- [CMSIS-MLEK](https://github.com/ARM-software/cmsis-mlek) GitHub repository +- [TensorFlow Lite](https://www.keil.arm.com/packs/tensorflow-lite-micro-tensorflow) Software Pack diff --git a/template/audio/kws/.clangd b/template/audio/kws/.clangd index 6100781..1e55abb 100644 --- a/template/audio/kws/.clangd +++ b/template/audio/kws/.clangd @@ -1,3 +1,2 @@ CompileFlags: - CompilationDatabase: >- - /Users/chrsei01/02_Git/Arm-Examples/cmsis-mlek/template/audio/out/kws/${Name}/Debug-Live_Stream + CompilationDatabase: c:\w\cmsis-mlek\template\audio\out\kws\${Name}\Debug-Live_Stream diff --git a/template/audio/vcpkg-configuration.json b/template/audio/vcpkg-configuration.json new file mode 100644 index 0000000..d11f445 --- /dev/null +++ b/template/audio/vcpkg-configuration.json @@ -0,0 +1,16 @@ +{ + "registries": [ + { + "name": "arm", + "kind": "artifact", + "location": "https://artifacts.tools.arm.com/vcpkg-registry" + } + ], + "requires": { + "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.10.0", + "arm:compilers/arm/armclang": "^6.23.0", + "arm:models/arm/avh-fvp": "^11.28.32", + "arm:tools/kitware/cmake": "^3.31.5", + "arm:tools/ninja-build/ninja": "^1.12.0" + } +} \ No newline at end of file diff --git a/template/generic/vcpkg-configuration.json b/template/generic/vcpkg-configuration.json index 9a47057..d11f445 100644 --- a/template/generic/vcpkg-configuration.json +++ b/template/generic/vcpkg-configuration.json @@ -7,13 +7,10 @@ } ], "requires": { - "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.8.0", + "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.10.0", "arm:compilers/arm/armclang": "^6.23.0", "arm:models/arm/avh-fvp": "^11.28.32", "arm:tools/kitware/cmake": "^3.31.5", - "arm:tools/ninja-build/ninja": "^1.12.0", - "arm:debuggers/arm/armdbg": "^6.4.0", - "arm:tools/arm/mdk-toolbox": "1.1.0", - "arm:compilers/arm/arm-none-eabi-gcc": "14.2.1" + "arm:tools/ninja-build/ninja": "^1.12.0" } } \ No newline at end of file diff --git a/template/video/vcpkg-configuration.json b/template/video/vcpkg-configuration.json index ec3da67..d11f445 100644 --- a/template/video/vcpkg-configuration.json +++ b/template/video/vcpkg-configuration.json @@ -7,11 +7,10 @@ } ], "requires": { - "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.8.0", + "arm:tools/open-cmsis-pack/cmsis-toolbox": "^2.10.0", "arm:compilers/arm/armclang": "^6.23.0", "arm:models/arm/avh-fvp": "^11.28.32", "arm:tools/kitware/cmake": "^3.31.5", - "arm:tools/ninja-build/ninja": "^1.12.0", - "arm:debuggers/arm/armdbg": "^6.4.0" + "arm:tools/ninja-build/ninja": "^1.12.0" } } \ No newline at end of file