diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..4dec4e1b1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,33 @@ +--- +name: Bug report +about: Report a bug you've encountered for further investigation +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Development Environment:** + - Linux Distribution: [e.g. Ubuntu20.04, RHEL8] + - Omniperf Version: [e.g. try `omniperf --version`] + - GPU: [e.g. Mi100, Mi200] + - Custer (if applicable): [e.g. Crusher, ] + +**To Reproduce** +Steps to reproduce the behavior: +1. Run '...' +2. Go to '...' +2. Click on '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..11fc491ef --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/src/docs/analysis.md b/src/docs/analysis.md index 9feff1f64..77ef92ef4 100644 --- a/src/docs/analysis.md +++ b/src/docs/analysis.md @@ -497,7 +497,7 @@ Each workload is imported to a separate database with the following naming conve e.g., omniperf_asw_vcopy_mi200. -Below is the sample command to import the *vcopy* profiling data. +When using database mode, be sure to tailor the connection options to the machine hosting your [sever-side instance](./installation.md). Below is the sample command to import the *vcopy* profiling data, lets assuming our host machine is called "dummybox". ```shell-session $ omniperf database --help diff --git a/src/docs/conf.py b/src/docs/conf.py index af0003fb7..09b6e1c2e 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -54,7 +54,9 @@ def install(package): myst_heading_anchors = 2 # enable replacement of (tm) & friends -myst_enable_extensions = ["replacements"] +myst_enable_extensions = [ + "replacements" +] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -90,8 +92,8 @@ def install(package): pygments_style = None # options for latex output -latex_engine = "lualatex" -latex_show_urls = "footnote" +latex_engine = 'lualatex' +latex_show_urls = 'footnote' # -- Options for HTML output ------------------------------------------------- diff --git a/src/docs/high_level_design.md b/src/docs/high_level_design.md index 6168b7ace..6d7f2a422 100644 --- a/src/docs/high_level_design.md +++ b/src/docs/high_level_design.md @@ -15,5 +15,7 @@ The [Omniperf](https://github.com/AMDResearch/omniperf) Tool is architecturally - *Grafana GUI Analyzer*: A Grafana dashboard is designed to retrieve the raw counters info from the backend database. It also creates the relevant performance metrics and visualization. - **Omniperf Standalone GUI Analyzer**: A standalone GUI is provided to enable performance analysis without importing data into the backend database. -![Omniperf Architectual Diagram](images/omniperf_architecture.png) +![Omniperf Architectual Diagram](images/omniperf_server_vs_client_install.png) + +> Note: To learn more about the client vs. server model of Omniperf and our install process please see the [Deployment section](./installation.md) of the docs. diff --git a/src/docs/images/datasource_config.jpg b/src/docs/images/datasource_config.jpg new file mode 100644 index 000000000..4210d9036 Binary files /dev/null and b/src/docs/images/datasource_config.jpg differ diff --git a/src/docs/images/datasource_config.png b/src/docs/images/datasource_config.png deleted file mode 100644 index 8e22ef737..000000000 Binary files a/src/docs/images/datasource_config.png and /dev/null differ diff --git a/src/docs/images/datasource_settings.jpg b/src/docs/images/datasource_settings.jpg new file mode 100644 index 000000000..f47236254 Binary files /dev/null and b/src/docs/images/datasource_settings.jpg differ diff --git a/src/docs/images/datasource_settings.png b/src/docs/images/datasource_settings.png deleted file mode 100644 index 04cf1e54d..000000000 Binary files a/src/docs/images/datasource_settings.png and /dev/null differ diff --git a/src/docs/images/grafana_workload_selection.png b/src/docs/images/grafana_workload_selection.png new file mode 100644 index 000000000..3ecdc35e7 Binary files /dev/null and b/src/docs/images/grafana_workload_selection.png differ diff --git a/src/docs/images/install_decision_tree.png b/src/docs/images/install_decision_tree.png new file mode 100644 index 000000000..1c62fba87 Binary files /dev/null and b/src/docs/images/install_decision_tree.png differ diff --git a/src/docs/images/omniperf_server_vs_client_install.png b/src/docs/images/omniperf_server_vs_client_install.png new file mode 100644 index 000000000..8c43dba9e Binary files /dev/null and b/src/docs/images/omniperf_server_vs_client_install.png differ diff --git a/src/docs/images/opening_dashboard.png b/src/docs/images/opening_dashboard.png new file mode 100644 index 000000000..5e6c7ea62 Binary files /dev/null and b/src/docs/images/opening_dashboard.png differ diff --git a/src/docs/installation.md b/src/docs/installation.md index e550669b7..9e32d3032 100644 --- a/src/docs/installation.md +++ b/src/docs/installation.md @@ -16,11 +16,15 @@ Omniperf is broken into two installation components: 2. **Omniperf Server-side (_Optional_)** - Mongo DB backend + Grafana instance +Determine what you need to install based on how you'd like to interact with Omniperf. See the decision tree below to help determine what installation is right for you. + +![Omniperf Installtion Decision Tree](images/install_decision_tree.png) + --- ## Client-side Installation -Omniperf requires the following basic software dependencies prior to usage: +Omniperf client-side requires the following basic software dependencies prior to usage: * Python (>=3.7) * CMake (>= 3.19) @@ -149,16 +153,19 @@ wishes to use instead. ## Server-side Setup -Note: Server-side setup is not required to profile or analyze -performance data from the CLI. It is provided as an additional mechanism to import performance -data for examination within a detailed [Grafana](https://github.com/grafana/grafana) GUI. +> Note: Server-side setup is not required to profile or analyze performance data from the CLI. It is provided as an additional mechanism to import performance data for examination within a detailed [Grafana](https://github.com/grafana/grafana) GUI. + +Omniperf server-side requires the following basic software dependencies prior to usage: + +* [Docker Engine](https://docs.docker.com/engine/install/) -The recommended process for enabling the server-side of Omniperf is to -use the provided Docker file to build the Grafana and MongoDB -instance. +The recommended process for enabling the server-side of Omniperf is to use the provided Docker file to build the Grafana and MongoDB instance. -### Install MongoDB Utils +Once you've decided which machine you'd like to use to host the Grafana and MongoDB instance, please follow the set up instructions below. + +### 1) Install MongoDB Utils Omniperf uses [mongoimport](https://www.mongodb.com/docs/database-tools/mongoimport/) to upload data to Grafana's backend database. Install for Ubuntu 20.04 is as follows: + ```bash $ wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb $ sudo apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb @@ -176,14 +183,16 @@ $ sudo docker volume create --driver local --opt type=none --opt device=/usr/loc $ sudo docker volume create --driver local --opt type=none --opt device=/usr/local/persist/mongodb --opt o=bind grafana-mongo-db ``` -### Build and Launch +### 3) Build and Launch + +We're now ready to build our Docker file. Navigate to your Omniperf install directory to begin. ```bash $ sudo docker-compose build $ sudo docker-compose up -d ``` > Note that TCP ports for Grafana (4000) and MongoDB (27017) in the docker container are mapped to 14000 and 27018, respectively, on the host side. -### Setup Grafana Instance +### 4) Setup Grafana Instance Once you've launced your docker container you should be able to reach Grafana at **http://\:14000**. The default login credentials for the first-time Grafana setup are: - Username: **admin** @@ -193,26 +202,41 @@ Once you've launced your docker container you should be able to reach Grafana at MongoDB Datasource Configuration -The MongoDB Datasource shall be configured prior to the first-time use. Navigate to Grafana's Configuration page (shown below) to add the **Omniperf Data** connection. +The MongoDB Datasource must be configured prior to the first-time use. Navigate to Grafana's Configuration page (shown below) to add the **Omniperf Data** connection. -![Omniperf Datasource Config](images/datasource_config.png) +![Omniperf Datasource Config](images/datasource_config.jpg) -Configure the following fields in the datasource: +Configure the following fields in the datasource settings: - HTTP URL: set to *http://localhost:3333* - MongoDB URL: set to *mongodb://temp:temp123@\:27018/admin?authSource=admin* - Database Name: set to *admin* -After properly configuring these fields click **Save & Test** to make sure your connection is successful. +After properly configuring these fields click **Save & Test** (as shown below) to make sure your connection is successful. > Note to avoid potential DNS issue, one may need to use the actual IP address for the host node in the MongoDB URL. -![Datasource Settings](images/datasource_settings.png) +![Datasource Settings](images/datasource_settings.jpg) Omniperf Dashboard Import -From *Create* → *Import*, (as seen below) upload the dashboard file, `/dashboards/Omniperf_v{__VERSION__}_pub.json`, from the Omniperf tarball. +From *Create* → *Import*, (as shown below) upload the dashboard file, `/dashboards/Omniperf_v{__VERSION__}_pub.json`, from the Omniperf tarball. Edit both the Dashboard Name and the Unique Identifier (UID) to uniquely identify the dashboard he/she will use. Click Import to finish the process. -![Import Dashboard](images/import_dashboard.png) \ No newline at end of file +![Import Dashboard](images/import_dashboard.png) + +Using your dashboard + +Once you've imported a dashboard you're ready to begin! Start by browsing availible dashboards and selecting the dashboard you've just imported. + +![Opening your dashboard](images/opening_dashboard.png) + +Remeber, you'll need to upload workload data to the DB backend before analyzing in your Grafana interface. We provide a detailed example of this in our [Analysis section](./analysis.md#grafana-gui-import). + +After a workload has been successfully uploaded, you should be able to select it from the workload dropdown located at the top of your Grafana dashboard. + +![Selecting Grafana workload](images/grafana_workload_selection.png) + +For more information on how to use the Grafana interface for anlysis please see the [Grafana section](./analysis.md#grafana-based-gui) in the Analyze Mode tab. + diff --git a/src/docs/profiling.md b/src/docs/profiling.md index b0b56dc91..da77f6d3a 100644 --- a/src/docs/profiling.md +++ b/src/docs/profiling.md @@ -232,20 +232,28 @@ drwxrwxr-x 2 colramos colramos 4096 Apr 11 16:42 perfmon ``` ### Filtering -To reduce profiling time and the counters collected one may use profiling filters. +To reduce profiling time and the counters collected one may use profiling filters. Profiling filters and their functionality depend on the underlying profiler being used. While Omniperf is profiler agnostic, we've provided a detailed description of profiling filters available when using Omniperf with [rocProfiler](https://rocm.docs.amd.com/projects/rocprofiler/en/latest/rocprof.html) below. + + Filtering Options: -- The `-k` \ flag allows for kernel filtering, which is compatible with the current rocprof utility. +- The `-k` \ flag allows for kernel filtering. Useage is equivalent with the current rocprof utility (see details below). -- The `-d` \ flag allows for dispatch ID filtering, which is compatible with the current rocprof utility. +- The `-d` \ flag allows for dispatch ID filtering. Useage is equivalent with the current rocprof utility (see details below). - The `-b` \ allows system profiling on one or more selected IP blocks to speed up the profiling process. One can gradually incorporate more IP blocks, without overwriting performance data acquired on other IP blocks. +```{note} +Be cautious while combining different profiling filters in the same call. Conflicting filters may result in error. + +i.e. filtering dispatch X, but dispatch X does not match your kernel name filter +``` + #### IP Block Filtering One can profile a selected IP Block to speed up the profiling process. All profiling results are accumulated in the same target directory, without overwriting those for other IP blocks, hence enabling the incremental profiling and analysis. -The following example only profiles SQ and TCC, skipping all other IP Blocks. +The following example only gathers hardware counters for SQ and TCC, skipping all other IP Blocks: ```shell $ omniperf profile --name vcopy -b SQ TCC -- ./sample/vcopy 1048576 256 Resolving rocprof @@ -283,7 +291,9 @@ Log: /home/colramos/GitHub/omniperf-pub/workloads/vcopy/mi200/log.txt ``` #### Kernel Filtering -The following example demonstrates profiling on selected kernels: +Kernel filtering is based on the name of the kernel(s) you'd like to isolate. Use a kernel name substring list to isolate desired kernels. + +The following example demonstrates profiling isolating the kernel matching substring "vecCopy": ```shell $ omniperf profile --name vcopy -k vecCopy -- ./vcopy 1048576 256 Resolving rocprof diff --git a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml index 986b2f0ae..cde2fab9e 100644 --- a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml @@ -107,7 +107,7 @@ Panel Config: LDS BW: value: AVG(((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs))) - unit: GB/sec + unit: Gb/s peak: (($sclk * $numCU) * 0.128) pop: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) diff --git a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml index 986b2f0ae..cde2fab9e 100644 --- a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml @@ -107,7 +107,7 @@ Panel Config: LDS BW: value: AVG(((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs))) - unit: GB/sec + unit: Gb/s peak: (($sclk * $numCU) * 0.128) pop: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) diff --git a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml index c197c0fc5..5d207c77e 100644 --- a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml @@ -124,7 +124,7 @@ Panel Config: LDS BW: value: AVG(((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs))) - unit: GB/sec + unit: Gb/s peak: (($sclk * $numCU) * 0.128) pop: AVG((((((SQ_LDS_IDX_ACTIVE - SQ_LDS_BANK_CONFLICT) * 4) * TO_INT($LDSBanks)) / (EndNs - BeginNs)) / (($sclk * $numCU) * 0.00128))) diff --git a/src/omniperf_analyze/utils/gui.py b/src/omniperf_analyze/utils/gui.py index ca05bd3ea..b5c2d5151 100644 --- a/src/omniperf_analyze/utils/gui.py +++ b/src/omniperf_analyze/utils/gui.py @@ -287,6 +287,88 @@ def build_bar_chart(display_df, table_config, norm_filt): def build_table_chart( display_df, table_config, original_df, display_columns, comparable_columns, decimal ): + if "Unit" in display_df: + what = display_df["Unit"] + if "Gb/s" in display_df["Unit"].values: + for idx, row in display_df[display_df["Unit"] == "Gb/s"].items(): + for curr_metric in row: + curr_row = display_df[display_df["Metric"] == curr_metric] + if not curr_row.empty: + if "Value" in curr_row: + if isinstance( + curr_row["Value"][0], + float, + ): + if curr_row.Value[0] < 0.001: + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Unit", + ] = "Kb/s" + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Value", + ] = ( + 1000000 * curr_row.Value + ) + elif curr_row.Value[0] < 1: + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Unit", + ] = "Mb/s" + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Value", + ] = ( + 1000 * curr_row.Value + ) + elif "Avg" in curr_row: + if isinstance(curr_row["Avg"][0], float): + if curr_row.Avg[0] < 0.001: + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Unit", + ] = "Kb/s" + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Avg", + ] = ( + 1000000 * curr_row.Avg + ) + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Min", + ] = ( + 1000000 * curr_row.Min + ) + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Max", + ] = ( + 1000000 * curr_row.Max + ) + elif curr_row.Avg[0] < 1: + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Unit", + ] = "Mb/s" + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Avg", + ] = ( + 1000 * curr_row.Avg + ) + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Min", + ] = ( + 1000 * curr_row.Min + ) + display_df.loc[ + (display_df["Metric"] == curr_metric), + "Max", + ] = ( + 1000 * curr_row.Max + ) d_figs = [] # build comlumns/header with formatting diff --git a/src/omniperf_analyze/utils/tty.py b/src/omniperf_analyze/utils/tty.py index d04dc2cb9..e429bc1d0 100644 --- a/src/omniperf_analyze/utils/tty.py +++ b/src/omniperf_analyze/utils/tty.py @@ -106,6 +106,7 @@ def show_all(args, runs, archConfigs, output): float(x) if x != "" else float(0) for x in base_df[header] ] + # insert unit fix here cur_df[header] = [ float(x) if x != "" else float(0) for x in cur_df[header] @@ -141,13 +142,175 @@ def show_all(args, runs, archConfigs, output): df = pd.concat([df, t_df], axis=1) else: + # insert unit fix here cur_df[header] = [ round(float(x), args.decimal) if x != "" else x for x in base_df[header] ] - + if "Unit" in cur_df.columns: + for idx, row in cur_df[ + cur_df["Unit"] == "Gb/s" + ].items(): + for curr_metric in row: + curr_row = cur_df[ + cur_df["Metric"] == curr_metric + ] + if not curr_row.empty: + if "Value" in curr_row: + if isinstance( + curr_row["Value"][0], + float, + ): + if ( + curr_row.Value[0] + < 0.001 + ): + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Unit", + ] = "Kb/s" + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Value", + ] = ( + 1000000 + * curr_row.Value + ) + elif ( + curr_row.Value[0] < 1 + ): + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Unit", + ] = "Mb/s" + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Value", + ] = ( + 1000 + * curr_row.Value + ) + elif "Avg" in curr_row: + if isinstance( + curr_row["Avg"][0], float + ): + if ( + curr_row.Avg[0] + < 0.001 + ): + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Unit", + ] = "Kb/s" + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Avg", + ] = ( + 1000000 + * curr_row.Avg + ) + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Min", + ] = ( + 1000000 + * curr_row.Min + ) + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Max", + ] = ( + 1000000 + * curr_row.Max + ) + elif curr_row.Avg[0] < 1: + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Unit", + ] = "Mb/s" + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Avg", + ] = ( + 1000 + * curr_row.Avg + ) + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Min", + ] = ( + 1000 + * curr_row.Min + ) + cur_df.loc[ + ( + cur_df[ + "Metric" + ] + == curr_metric + ), + "Max", + ] = ( + 1000 + * curr_row.Max + ) df = pd.concat([df, cur_df[header]], axis=1) if not df.empty: