Tatsuya-hasegawa · Tatsuya-hasegawa · Jul 27, 2023 · Apr 21, 2023 · May 10, 2023 · May 10, 2023
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -28,7 +28,7 @@ jobs:
           JOB_CONTEXT: ${{ toJSON(job) }}
         run: echo "$JOB_CONTEXT"
       # end print details
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -82,7 +82,7 @@ jobs:
           pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml
         if: ${{ always() }}
       - name: Upload pytest test results
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: pytest-results-${{ matrix.python-version }}
           path: junit/test-${{ matrix.python-version }}-results.xml
@@ -95,7 +95,7 @@ jobs:
       matrix:
         python-version: ["3.8"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -159,7 +159,7 @@ jobs:
           mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy
         if: ${{ always() }}
       - name: Upload mypy test results
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: Mypy results ${{ matrix.python-version }}
           path: junit/mypy-test-${{ matrix.python-version }}-results.xml

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -11,6 +11,7 @@ name: Upload Python Package to PyPI Prod
 on:
   release:
     types: [published]
+  workflow_dispatch:
 
 permissions:
   contents: read

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -52,4 +52,4 @@ repos:
         entry: python -m tools.create_reqs_all
         pass_filenames: False
         language: python
-        types: [python]
+        types: [python]
diff --git a/conda/conda-reqs.txt b/conda/conda-reqs.txt
@@ -4,22 +4,24 @@ azure-core>=1.24.0
 azure-mgmt-core>=1.2.1
 azure-identity>=1.10.0
 azure-keyvault-secrets>=4.0.0
+azure-kusto-data>=4.0.0
 azure-mgmt-compute>=4.6.2
 azure-mgmt-keyvault>=2.0.0
 azure-mgmt-network>=2.7.0
 azure-mgmt-resource>=16.1.0
+azure-monitor-query>=1.0.0
 azure-storage-blob>=12.5.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <=2.4.3
+bokeh>=1.4.0, <4.0.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0
 folium>=0.9.0
 geoip2>=2.9.0
 html5lib
-httpx==0.23.3
+httpx==0.24.1
 ipython>=7.23.1
-ipywidgets>=7.4.2, <8.0.0
+ipywidgets>=7.4.2, <9.0.0
 keyring>=13.2.1
 lxml>=4.6.5
 matplotlib>=3.0.0

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -2,7 +2,7 @@ attrs>=18.2.0
 cryptography
 deprecated>=1.2.4
 docutils<0.20.0
-httpx==0.23.3
+httpx==0.24.1
 ipython >= 7.1.1
 jinja2<3.2.0
 numpy>=1.15.4
@@ -11,7 +11,8 @@ python-dateutil>=2.8.1
 pytz>=2019.2
 pyyaml>=3.13
 typing-extensions>=4.2.0
-readthedocs-sphinx-ext==2.2.0
+readthedocs-sphinx-ext==2.2.2
 seed_intersphinx_mapping
-sphinx-rtd-theme==1.2.0
+sphinx-rtd-theme==1.2.2
 sphinx==6.1.3
+sphinxcontrib-jquery==4.1
diff --git a/docs/source/DataAcquisition.rst b/docs/source/DataAcquisition.rst
@@ -17,6 +17,7 @@ Individual Data Environments
    :maxdepth: 2
 
    data_acquisition/DataProv-MSSentinel
+   data_acquisition/DataProv-MSSentinel-New
    data_acquisition/DataProv-MSDefender
    data_acquisition/DataProv-MSGraph
    data_acquisition/DataProv-LocalData
@@ -25,8 +26,10 @@ Individual Data Environments
    data_acquisition/MordorData
    data_acquisition/DataProv-Sumologic
    data_acquisition/DataProv-Kusto
+   data_acquisition/DataProv-Kusto-New
    data_acquisition/DataProv-Cybereason
    data_acquisition/DataProv-OSQuery
+   data_acquisition/DataProv-Velociraptor
 
 
 Built-in Data Queries
@@ -54,4 +57,4 @@ Contributing a Data Provider
 .. toctree::
    :maxdepth: 2
 
-   data_acquisition/WritingADataProvider
+   extending/WritingDataProviders
diff --git a/docs/source/Development.rst b/docs/source/Development.rst
@@ -0,0 +1,94 @@
+MSTICPy Development Guidelines
+==============================
+
+Contributions of improvements, fixes and new features are all
+welcomed. Whether this is your first time contributing to a
+project or you are a seasoned Open-Source contributor,
+we welcome your contribution. In this guide you can find a few
+pointers to help you create a great contribution.
+
+What to contribute
+------------------
+
+There are many things that can make a good contribution.
+It might be a fix for a specific issue you have come across,
+an improvement to an existing feature that you have thought
+about such as a new data connector or threat intelligence provider,
+or a completely new feature category.
+
+If you don't have a specific idea in mind take a look at the
+`Issues page on GitHub <https://github.com/microsoft/msticpy/issues>`__
+
+This page tracks a range of issues, enhancements, and features that
+members of the community have thought of. The MSTICPy team uses these
+issues as a way to track work and includes many things we have added ourselves.
+
+The issues are tagged with various descriptions that relate to the
+type of issue. You may see some with the ‘good first issue’ tag.
+These are issues that we think would make a good issue for someone
+contributing to MSTICPy for the first time, however anyone is welcome
+to work on any Issue. If you decide to start working on an Issue please
+make a comment on the Issue so that we can assign it to you and other
+members of the community know that it is being worked on and don’t
+duplicate work. Also if you are unclear about what the Issue feel
+free to comment on the Issue to get clarification from others.
+
+
+
+What makes a good contribution?
+-------------------------------
+
+Whilst there is no one thing that makes a contribution good here are some guidelines:
+
+Scope
+~~~~~
+Focus your contribution on a single thing per PR (Pull Request) raised, whether it
+be a feature or a fix. If you have multiple things you want to contribute,
+consider splitting them into multiple PRs. Keeping each PR to a single item
+makes it easier for others to see what you are contributing and how it
+fits with the rest of the project.
+
+Documentation
+-------------
+Make it clear what you are contributing, why its important, and how
+it works. This provides much needed clarity for others when reviewing
+contributions and helps to highlight the great value in your contribution.
+
+Unit test and test Coverage
+---------------------------
+Write unit tests for your code. We use `pytest <https://pytest.org>`__
+to run our tests.
+
+See the section :ref:`dev/CodingGuidelines:Unit Tests` for more information.
+
+Using Git
+---------
+To contribute you will need to fork the MSTICPy repo.
+**Create a branch** for your contribution, make the code changes
+and then raise a PR to merge the changes back into
+MSTICPy's main branch. Please *do not* make changes to `main` of your
+fork and submit this as a PR.
+You should also consider granting permission on your fork so that
+we can push changes back to your forked branch. Sometimes, it's
+quicker for us to make a quick change to fix something than to ask
+you to make the change. If we cannot push any changes back
+this is impossible to do.
+
+If you are unfamiliar with Git and GitHub you can find some
+guidance here: https://docs.github.com/en/get-started/quickstart/set-up-git
+
+
+Where to get help
+-----------------
+We are more than happy to help support your contributions,
+if you need help you can comment on the Issue you are working on,
+or email [msticpy@microsoft.com](mailto:msticpy@microsoft.com)
+
+You can also join our Discord
+`#msticpy <https://discordapp.com/channels/717911137915764877/922881584288399410>`.
+
+
+.. toctree::
+   :maxdepth: 2
+
+   dev/CodingGuidelines
diff --git a/docs/source/ExtendingMsticpy.rst b/docs/source/ExtendingMsticpy.rst
@@ -0,0 +1,35 @@
+Extending MSTICPy
+=================
+
+Introduction to MSTICPy extensibility
+-------------------------------------
+
+MSTICPy has several extensibility points. These range from adding
+parameterized queries to writing your own data provider or
+context provider.
+
+Some of these require coding, while others can be done
+by creating YAML configuration files. For Data Providers and
+Context/TI providers there is also a plugin model that allows
+you to create private providers and load them from a local
+path.
+
+
+Contributing
+------------
+
+If you decide to extend MSTICPy in one of these ways and
+think that this would be useful to other users of the
+package, please consider contributing them into the package.
+
+Extension points documentation
+------------------------------
+
+.. toctree::
+   :maxdepth: 2
+
+   extending/Queries
+   extending/PivotFunctions
+   extending/WritingDataProviders
+   extending/WritingTIAndContextProviders
+   extending/PluginFramework
diff --git a/docs/source/api/msticpy.analysis.polling_detection.rst b/docs/source/api/msticpy.analysis.polling_detection.rst
@@ -0,0 +1,7 @@
+msticpy.analysis.polling\_detection module
+==========================================
+
+.. automodule:: msticpy.analysis.polling_detection
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.analysis.rst b/docs/source/api/msticpy.analysis.rst
@@ -25,5 +25,6 @@ Submodules
    msticpy.analysis.eventcluster
    msticpy.analysis.observationlist
    msticpy.analysis.outliers
+   msticpy.analysis.polling_detection
    msticpy.analysis.syslog_utils
    msticpy.analysis.timeseries
diff --git a/docs/source/api/msticpy.common.proxy_settings.rst b/docs/source/api/msticpy.common.proxy_settings.rst
@@ -0,0 +1,7 @@
+msticpy.common.proxy\_settings module
+=====================================
+
+.. automodule:: msticpy.common.proxy_settings
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.common.rst b/docs/source/api/msticpy.common.rst
@@ -27,5 +27,7 @@ Submodules
    msticpy.common.exceptions
    msticpy.common.pkg_config
    msticpy.common.provider_settings
+   msticpy.common.proxy_settings
+   msticpy.common.settings
    msticpy.common.timespan
    msticpy.common.wsconfig
diff --git a/docs/source/api/msticpy.common.settings.rst b/docs/source/api/msticpy.common.settings.rst
@@ -0,0 +1,7 @@
+msticpy.common.settings module
+==============================
+
+.. automodule:: msticpy.common.settings
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.config.ce_msticpy.rst b/docs/source/api/msticpy.config.ce_msticpy.rst
@@ -0,0 +1,7 @@
+msticpy.config.ce\_msticpy module
+=================================
+
+.. automodule:: msticpy.config.ce_msticpy
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.config.rst b/docs/source/api/msticpy.config.rst
@@ -17,6 +17,7 @@ Submodules
    msticpy.config.ce_common
    msticpy.config.ce_data_providers
    msticpy.config.ce_keyvault
+   msticpy.config.ce_msticpy
    msticpy.config.ce_other_providers
    msticpy.config.ce_provider_base
    msticpy.config.ce_simple_settings

diff --git a/docs/source/api/msticpy.data.core.query_provider_connections_mixin.rst b/docs/source/api/msticpy.data.core.query_provider_connections_mixin.rst
@@ -0,0 +1,7 @@
+msticpy.data.core.query\_provider\_connections\_mixin module
+============================================================
+
+.. automodule:: msticpy.data.core.query_provider_connections_mixin
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.core.query_provider_utils_mixin.rst b/docs/source/api/msticpy.data.core.query_provider_utils_mixin.rst
@@ -0,0 +1,7 @@
+msticpy.data.core.query\_provider\_utils\_mixin module
+======================================================
+
+.. automodule:: msticpy.data.core.query_provider_utils_mixin
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.core.query_template.rst b/docs/source/api/msticpy.data.core.query_template.rst
@@ -0,0 +1,7 @@
+msticpy.data.core.query\_template module
+========================================
+
+.. automodule:: msticpy.data.core.query_template
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.core.rst b/docs/source/api/msticpy.data.core.rst
@@ -17,5 +17,8 @@ Submodules
    msticpy.data.core.param_extractor
    msticpy.data.core.query_container
    msticpy.data.core.query_defns
+   msticpy.data.core.query_provider_connections_mixin
+   msticpy.data.core.query_provider_utils_mixin
    msticpy.data.core.query_source
    msticpy.data.core.query_store
+   msticpy.data.core.query_template
diff --git a/docs/source/api/msticpy.data.drivers.azure_kusto_driver.rst b/docs/source/api/msticpy.data.drivers.azure_kusto_driver.rst
@@ -0,0 +1,7 @@
+msticpy.data.drivers.azure\_kusto\_driver module
+================================================
+
+.. automodule:: msticpy.data.drivers.azure_kusto_driver
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.drivers.azure_monitor_driver.rst b/docs/source/api/msticpy.data.drivers.azure_monitor_driver.rst
@@ -0,0 +1,7 @@
+msticpy.data.drivers.azure\_monitor\_driver module
+==================================================
+
+.. automodule:: msticpy.data.drivers.azure_monitor_driver
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.drivers.local_velociraptor_driver.rst b/docs/source/api/msticpy.data.drivers.local_velociraptor_driver.rst
@@ -0,0 +1,7 @@
+msticpy.data.drivers.local\_velociraptor\_driver module
+=======================================================
+
+.. automodule:: msticpy.data.drivers.local_velociraptor_driver
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/api/msticpy.data.drivers.rst b/docs/source/api/msticpy.data.drivers.rst
@@ -12,17 +12,21 @@ Submodules
 .. toctree::
    :maxdepth: 4
 
+   msticpy.data.drivers.azure_kusto_driver
+   msticpy.data.drivers.azure_monitor_driver
    msticpy.data.drivers.cybereason_driver
    msticpy.data.drivers.driver_base
    msticpy.data.drivers.elastic_driver
    msticpy.data.drivers.kql_driver
    msticpy.data.drivers.kusto_driver
    msticpy.data.drivers.local_data_driver
    msticpy.data.drivers.local_osquery_driver
+   msticpy.data.drivers.local_velociraptor_driver
    msticpy.data.drivers.mdatp_driver
    msticpy.data.drivers.mordor_driver
    msticpy.data.drivers.odata_driver
    msticpy.data.drivers.resource_graph_driver
    msticpy.data.drivers.security_graph_driver
+   msticpy.data.drivers.sentinel_query_reader
    msticpy.data.drivers.splunk_driver
    msticpy.data.drivers.sumologic_driver
diff --git a/docs/source/api/msticpy.data.drivers.sentinel_query_reader.rst b/docs/source/api/msticpy.data.drivers.sentinel_query_reader.rst
@@ -0,0 +1,7 @@
+msticpy.data.drivers.sentinel\_query\_reader module
+===================================================
+
+.. automodule:: msticpy.data.drivers.sentinel_query_reader
+   :members:
+   :undoc-members:
+   :show-inheritance: