Merge pull request #8 from dagardner-nv/david_mdd_dfp-example-docs

Draft: DFP Example documentation
mdemoret-nv · Sep 17, 2022 · 9c17607 · 9c17607
2 parents 4c83dff + 04d9bf1
commit 9c17607
Show file tree

Hide file tree

Showing 17 changed files with 371 additions and 5 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,3 +1,4 @@
+docs/source/developer_guide/guides/img/** filter=lfs diff=lfs merge=lfs -text
 examples/data/** filter=lfs diff=lfs merge=lfs -text
 morpheus/_version.py export-subst
 tests/mock_triton_server/payloads/** filter=lfs diff=lfs merge=lfs -text

diff --git a/ci/scripts/jenkins/docs.sh b/ci/scripts/jenkins/docs.sh
@@ -21,6 +21,12 @@ source ${WORKSPACE}/ci/scripts/jenkins/common.sh
 restore_conda_env
 pip install ${MORPHEUS_ROOT}/build/wheel
 
+gpuci_logger "Pulling LFS assets"
+cd ${MORPHEUS_ROOT}
+
+git lfs install
+${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs
+
 cd ${MORPHEUS_ROOT}/docs
 gpuci_logger "Installing Documentation dependencies"
 pip install -r requirement.txt

diff --git a/docs/source/developer_guide/guides.rst b/docs/source/developer_guide/guides.rst
@@ -25,3 +25,4 @@ Morpheus includes several stages to choose from when building a custom pipeline,
    guides/2_real_world_phishing
    guides/3_simple_cpp_stage
    guides/4_source_cpp_stage
+   guides/5_digital_fingerprinting
diff --git a/docs/source/developer_guide/guides/5_digital_fingerprinting.md b/docs/source/developer_guide/guides/5_digital_fingerprinting.md
diff --git a/docs/source/developer_guide/guides/img/dfp_deployment_configs.png b/docs/source/developer_guide/guides/img/dfp_deployment_configs.png
diff --git a/docs/source/developer_guide/guides/img/dfp_high_level_arch.png b/docs/source/developer_guide/guides/img/dfp_high_level_arch.png
diff --git a/docs/source/developer_guide/guides/img/dfp_inference_overview.png b/docs/source/developer_guide/guides/img/dfp_inference_overview.png
diff --git a/docs/source/developer_guide/guides/img/dfp_input_config.png b/docs/source/developer_guide/guides/img/dfp_input_config.png
diff --git a/docs/source/developer_guide/guides/img/dfp_output_config.png b/docs/source/developer_guide/guides/img/dfp_output_config.png
diff --git a/docs/source/developer_guide/guides/img/dfp_pipeline_structure.png b/docs/source/developer_guide/guides/img/dfp_pipeline_structure.png
diff --git a/docs/source/developer_guide/guides/img/dfp_runtime_env.png b/docs/source/developer_guide/guides/img/dfp_runtime_env.png
diff --git a/docs/source/developer_guide/guides/img/dfp_training_overview.png b/docs/source/developer_guide/guides/img/dfp_training_overview.png
diff --git a/docs/source/developer_guide/guides/img/sink_deps.png b/docs/source/developer_guide/guides/img/sink_deps.png
diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_postprocessing_stage.py b/examples/digital_fingerprinting/production/morpheus/dfp/stages/dfp_postprocessing_stage.py
@@ -50,7 +50,6 @@ def accepted_types(self) -> typing.Tuple:
 
     def _extract_events(self, message: MultiAEMessage):
 
-        # Return the message for the next stage
         z_scores = message.get_meta("mean_abs_z")
 
         above_threshold_df = message.get_meta()[z_scores > self._z_score_threshold]

diff --git a/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py b/examples/digital_fingerprinting/production/morpheus/dfp_duo_pipeline.py
@@ -185,8 +185,6 @@ def run_pipeline(train_users,
                    true_values=["success", "SUCCESS"],
                    false_values=["denied", "DENIED", "FRAUD"]),
         ColumnInfo(name="reason", dtype=str),
-        # CustomColumn(name="user.groups", dtype=str, process_column_fn=partial(column_listjoin,
-        # col_name="user.groups"))
     ]
 
     source_schema = DataFrameInputSchema(json_columns=["access_device", "application", "auth_device", "user"],

diff --git a/examples/digital_fingerprinting/starter/README.md b/examples/digital_fingerprinting/starter/README.md
@@ -49,8 +49,7 @@ Usage: morpheus run pipeline-ae [OPTIONS] COMMAND1 [ARGS]... [COMMAND2
   4. The following stages must come after an inference stage: `add-class`, `filter`, `gen-viz`
 
 Options:
-  --columns_file FILE             [default: ./morpheus/data/columns_ae_cloudtrail.txt
-                                  ]
+  --columns_file FILE             [default: ./morpheus/data/columns_ae_cloudtrail.txt]
   --labels_file FILE              Specifies a file to read labels from in
                                   order to convert class IDs into labels. A
                                   label file is a simple text file where each

diff --git a/scripts/fetch_data.py b/scripts/fetch_data.py
@@ -23,6 +23,7 @@
 
 LFS_DATASETS = {
     'all': '**',
+    'docs': 'docs/**',
     'examples': 'examples/**',
     'models': 'models/**',
     'tests': 'tests/**',