Split data dir, moving large files into examples/data

.gitattributes

@@ -1,3 +1,4 @@
+examples/data/* filter=lfs diff=lfs merge=lfs -text
 morpheus/_version.py export-subst
 morpheus/data/* filter=lfs diff=lfs merge=lfs -text
 tests/expected_data/* filter=lfs diff=lfs merge=lfs -text

docs/source/basics/examples.rst

@@ -35,15 +35,15 @@ This example will copy the values from Kafka into ``out.jsonlines``.
 Remove Fields from JSON Objects
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This example will only copy the fields 'timestamp', 'src_ip' and 'dest_ip' from ``morpheus/data/pcap_dump.jsonlines`` to
+This example will only copy the fields 'timestamp', 'src_ip' and 'dest_ip' from ``examples/data/pcap_dump.jsonlines`` to
 ``out.jsonlines``.
 
 .. image:: img/remove_fields_from_json_objects.png
 
 .. code-block:: bash
 
    morpheus run pipeline-nlp --viz_file=basic_usage_img/remove_fields_from_json_objects.png \
-      from-file --filename morpheus/data/pcap_dump.jsonlines \
+      from-file --filename examples/data/pcap_dump.jsonlines \
       deserialize \
       serialize --include 'timestamp' --include 'src_ip' --include 'dest_ip' \
       to-file --filename out.jsonlines
@@ -58,7 +58,7 @@ This example will report the throughput on the command line.
 .. code-block:: console
 
    $ morpheus run pipeline-nlp --viz_file=basic_usage_img/monitor_throughput.png  \
-      from-file --filename morpheus/data/pcap_dump.jsonlines \
+      from-file --filename examples/data/pcap_dump.jsonlines \
       deserialize \
       monitor --description "Lines Throughput" --smoothing 0.1 --unit "lines" \
       serialize \
@@ -79,7 +79,7 @@ decouple one stage from the next. Without the buffers, all montioring would show
 .. code-block:: console
 
    $ morpheus run pipeline-nlp --viz_file=basic_usage_img/multi_monitor_throughput.png  \
-      from-file --filename morpheus/data/pcap_dump.jsonlines \
+      from-file --filename examples/data/pcap_dump.jsonlines \
       monitor --description "From File Throughput" \
       buffer \
       deserialize \
@@ -107,7 +107,7 @@ This example shows an NLP Pipeline which uses most stages available in Morpheus.
 
    $ morpheus run --num_threads=8 --pipeline_batch_size=1024 --model_max_batch_size=32 \
       pipeline-nlp --viz_file=basic_usage_img/nlp_kitchen_sink.png  \
-      from-file --filename morpheus/data/pcap_dump.jsonlines \
+      from-file --filename examples/data/pcap_dump.jsonlines \
       buffer --count=500 \
       deserialize \
       preprocess \

docs/source/developer_guide/guides/1_simple_python_stage.md

@@ -31,8 +31,8 @@ Defining this stage requires us to specify the stage type. Morpheus stages conta
 import typing
 
 import neo
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 class PassThruStage(SinglePortStage):
 ```
@@ -87,8 +87,8 @@ import typing
 
 import neo
 
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 class PassThruStage(SinglePortStage):
     @property
@@ -122,8 +122,8 @@ import os
 
 from morpheus.config import Config
 from morpheus.pipeline import LinearPipeline
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
 from morpheus.utils.logging import configure_logging
 
 from pass_thru import PassThruStage
@@ -183,8 +183,8 @@ import os
 
 from morpheus.config import Config
 from morpheus.pipeline import LinearPipeline
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.input.from_file import FileSourceStage
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
 from morpheus.utils.logging import configure_logging
 
 from pass_thru import PassThruStage

docs/source/developer_guide/guides/2_real_world_phishing.md

@@ -80,9 +80,9 @@ import typing
 
 import neo
 
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages.message_meta import MessageMeta
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 
 class RecipientFeaturesStage(SinglePortStage):
@@ -193,7 +193,7 @@ out_dir = os.environ.get('OUT_DIR', '/tmp')
 labels_file = os.path.join(morpheus.DATA_DIR, 'labels_phishing.txt')
 vocab_file = os.path.join(morpheus.DATA_DIR, 'bert-base-uncased-hash.txt')
 
-input_file = os.path.join(root_dir, 'examples/data/email.jsonlines')
+input_file = os.path.join(root_dir, 'examples/data/email_with_addresses.jsonlines')
 results_file = os.path.join(out_dir, 'detections.jsonlines')
 ```
 
@@ -290,14 +290,14 @@ import morpheus
 from morpheus.config import Config
 from morpheus.config import PipelineModes
 from morpheus.pipeline import LinearPipeline
-from morpheus.pipeline.general_stages import FilterDetectionsStage
-from morpheus.pipeline.general_stages import MonitorStage
-from morpheus.pipeline.inference.inference_triton import TritonInferenceStage
-from morpheus.pipeline.input.from_file import FileSourceStage
-from morpheus.pipeline.output.serialize import SerializeStage
-from morpheus.pipeline.output.to_file import WriteToFileStage
-from morpheus.pipeline.preprocessing import DeserializeStage
-from morpheus.pipeline.preprocessing import PreprocessNLPStage
+from morpheus.stages.general.monitor_stage import MonitorStage
+from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage
+from morpheus.stages.input.file_source_stage import FileSourceStage
+from morpheus.stages.output.write_to_file_stage import WriteToFileStage
+from morpheus.stages.postprocess.filter_detections_stage import FilterDetectionsStage
+from morpheus.stages.postprocess.serialize_stage import SerializeStage
+from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
+from morpheus.stages.preprocess.preprocess_nlp_stage import PreprocessNLPStage
 from morpheus.utils.logging import configure_logging
 
 from recipient_feature_stage import RecipientFeaturesStage
@@ -313,7 +313,7 @@ def run_pipeline():
     labels_file = os.path.join(morpheus.DATA_DIR, 'labels_phishing.txt')
     vocab_file = os.path.join(morpheus.DATA_DIR, 'bert-base-uncased-hash.txt')
 
-    input_file = os.path.join(root_dir, 'examples/data/email.jsonlines')
+    input_file = os.path.join(root_dir, 'examples/data/email_with_addresses.jsonlines')
     results_file = os.path.join(out_dir, 'detections.jsonlines')
 
     # It's necessary to configure the pipeline for NLP mode
@@ -453,9 +453,9 @@ import pika
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.pipeline import SingleOutputSource
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages.message_meta import MessageMeta
+from morpheus.pipeline.single_output_source import SingleOutputSource
+from morpheus.pipeline.stream_pair import StreamPair
 
 logger = logging.getLogger(__name__)
 
@@ -597,9 +597,9 @@ import pika
 import cudf
 
 from morpheus.config import Config
-from morpheus.pipeline.messages import MessageMeta
-from morpheus.pipeline.pipeline import SinglePortStage
-from morpheus.pipeline.pipeline import StreamPair
+from morpheus.messages.message_meta import MessageMeta
+from morpheus.pipeline.single_port_stage import SinglePortStage
+from morpheus.pipeline.stream_pair import StreamPair
 
 logger = logging.getLogger(__name__)
 

docs/source/morpheus_quickstart_guide.md

@@ -559,7 +559,7 @@ $ helm install --set ngc.apiKey="$API_KEY" \
       pipeline-nlp \
         --model_seq_length=128 \
         --labels_file=./morpheus/data/labels_phishing.txt \
-        from-file --filename=./morpheus/data/email.jsonlines \
+        from-file --filename=./examples/data/email.jsonlines \
         monitor --description 'FromFile Rate' --smoothing=0.001 \
         deserialize \
         preprocess --vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \
@@ -635,7 +635,7 @@ $ helm install --set ngc.apiKey="$API_KEY" \
       --model_max_batch_size=32 \
       pipeline-nlp \
         --model_seq_length=256 \
-        from-file --filename=./morpheus/data/pcap_dump.jsonlines \
+        from-file --filename=./examples/data/pcap_dump.jsonlines \
         monitor --description 'FromFile Rate' --smoothing=0.001 \
         deserialize \
         preprocess --vocab_hash_file=./morpheus/data/bert-base-uncased-hash.txt --truncation=True --do_lower_case=True --add_special_tokens=False \
@@ -685,7 +685,7 @@ Make sure you create input and output Kafka topics before you start the pipeline
 $ kubectl -n $NAMESPACE exec -it deploy/broker -c broker -- kafka-console-producer.sh \
        --broker-list broker:9092 \
        --topic <YOUR_INPUT_KAFKA_TOPIC> < \
-       <YOUR_INPUT_DATA_FILE_PATH_EXAMPLE: ${HOME}/morpheus/data/pcap_dump.jsonlines>
+       <YOUR_INPUT_DATA_FILE_PATH_EXAMPLE: ${HOME}/examples/data/pcap_dump.jsonlines>
 ```
 
 **Note**: This should be used for development purposes only via this developer kit. Loading from the file into Kafka should not be used in production deployments of Morpheus.
@@ -708,7 +708,7 @@ $ helm install --set ngc.apiKey="$API_KEY" \
         --model_max_batch_size=64 \
         --use_cpp=True \
         pipeline-fil \
-          from-file --filename=./morpheus/data/nvsmi.jsonlines \
+          from-file --filename=./examples/data/nvsmi.jsonlines \
           monitor --description 'FromFile Rate' --smoothing=0.001 \
           deserialize \
           preprocess \
@@ -754,7 +754,7 @@ Make sure you create input and output Kafka topics before you start the pipeline
 $ kubectl -n $NAMESPACE exec -it deploy/broker -c broker -- kafka-console-producer.sh \
        --broker-list broker:9092 \
        --topic <YOUR_INPUT_KAFKA_TOPIC> < \
-       <YOUR_INPUT_DATA_FILE_PATH_EXAMPLE: ${HOME}/morpheus/data/nvsmi.jsonlines>
+       <YOUR_INPUT_DATA_FILE_PATH_EXAMPLE: ${HOME}/examples/data/nvsmi.jsonlines>
 ```
 
 **Note**: This should be used for development purposes only via this developer kit. Loading from the file into Kafka should not be used in production deployments of Morpheus.

examples/abp_nvsmi_detection/README.md

@@ -46,7 +46,7 @@ $ nvidia-smi dmon
 
 Each line in the output represents the GPU metrics at a single point in time. As the tool progresses the GPU begins to be utilized and you can see the SM% and Mem% increase as memory is loaded into the GPU and computations are performed. The model we will be using can ingest this information and determine whether or not the GPU is mining cryptocurriences without needing additional information from the host machine.
 
-In this example we will be using the `morpheus/data/nvsmi.jsonlines` dataset that is known to contain mining behavior profiles. The dataset is in the `.jsonlines` format which means each new line represents an new JSON object. In order to parse this data, it must be ingested, split by lines into individual JSON objects, and parsed into cuDF dataframes. This will all be handled by Morpheus.
+In this example we will be using the `examples/data/nvsmi.jsonlines` dataset that is known to contain mining behavior profiles. The dataset is in the `.jsonlines` format which means each new line represents an new JSON object. In order to parse this data, it must be ingested, split by lines into individual JSON objects, and parsed into cuDF dataframes. This will all be handled by Morpheus.
 
 ## Pipeline Architecture
 
@@ -102,7 +102,7 @@ morpheus --log_level=DEBUG \
    `# Specify a NLP pipeline with 256 sequence length (Must match Triton config)` \
    pipeline-fil \
    `# 1st Stage: Read from file` \
-   from-file --filename=$MORPHEUS_ROOT/morpheus/data/nvsmi.jsonlines \
+   from-file --filename=$MORPHEUS_ROOT/examples/data/nvsmi.jsonlines \
    `# 2nd Stage: Deserialize from JSON strings to objects` \
    deserialize \
    `# 3rd Stage: Preprocessing converts the input data into BERT tokens` \
@@ -178,7 +178,7 @@ CPP Enabled: True
 ====Registering Pipeline Complete!====
 ====Starting Pipeline====
 ====Building Pipeline====
-Added source: <from-file-0; FileSourceStage(filename=/home/dagardner/work/morpheus/data/nvsmi.jsonlines, iterative=False, file_type=FileTypes.Auto, repeat=1, filter_null=True, cudf_kwargs=None)>
+Added source: <from-file-0; FileSourceStage(filename=/home/dagardner/work/examples/data/nvsmi.jsonlines, iterative=False, file_type=FileTypes.Auto, repeat=1, filter_null=True, cudf_kwargs=None)>
   └─> morpheus.MessageMeta
 Added stage: <deserialize-1; DeserializeStage()>
   └─ morpheus.MessageMeta -> morpheus.MultiMessage

examples/abp_pcap_detection/README.md

@@ -83,11 +83,11 @@ Options:
   --help                          Show this message and exit.
 ```
 
-To launch the configured Morpheus pipeline with the sample data that is provided at `<MORPHEUS_ROOT>/morpheus/data`, run the following:
+To launch the configured Morpheus pipeline with the sample data that is provided at `<MORPHEUS_ROOT>/examples/data`, run the following:
 
 ```bash
 python run.py \
-	--input_file ../../morpheus/data/abp_pcap_dump.jsonlines \
+	--input_file ../data/abp_pcap_dump.jsonlines \
 	--output_file ./pcap_out.jsonlines \
 	--model_name 'abp-pcap-xgb' \
 	--server_url localhost:8001