353- initial infra for fideslog integration

CHANGELOG.md

@@ -54,6 +54,7 @@ The types of changes are:
 * DRP exerise endpoint [#496](https://github.com/ethyca/fidesops/pull/496)
 * Frontend for privacy request denial reaons [#480](https://github.com/ethyca/fidesops/pull/480)
 * Publish Fidesops to Pypi [#491](https://github.com/ethyca/fidesops/pull/491)
+* ADDS Fideslog integration [#541](https://github.com/ethyca/fidesops/pull/541)
 
 
 ### Changed 

docs/fidesops/docs/guides/configuration_reference.md

@@ -53,6 +53,9 @@ The `fidesops.toml` file should specify the following variables:
 |`TASK_RETRY_BACKOFF` | `FIDESOPS__EXECUTION__TASK_RETRY_BACKOFF` | int | 2 | 1 | The backoff factor for retries, to space out repeated retries.
 |`REQUIRE_MANUAL_REQUEST_APPROVAL` | `FIDESOPS__EXECUTION__REQUIRE_MANUAL_REQUEST_APPROVAL` | bool | False | False | Whether privacy requests require explicit approval to execute
 |`MASKING_STRICT` | `FIDESOPS__EXECUTION__MASKING_STRICT` | bool | True | True | If MASKING_STRICT is True, we only use "update" requests to mask data. (For third-party integrations, you should define an `update` endpoint to use.)  If MASKING_STRICT is False, you are allowing fidesops to use any defined DELETE or GDPR DELETE endpoints to remove PII. In this case, you should define `delete` or `data_protection_request` endpoints for your third-party integrations.  Note that setting MASKING_STRICT to False means that data may be deleted beyond the specific data categories that you've configured in your Policy.
+|---|---|---|---|---|---|
+|`ANALYTICS_OPT_OUT` | `FIDESOPS__USER__ANALYTICS_OPT_OUT` | bool | True | True | Opt out of sending anonymous usage data to Ethyca to improve the product experience.
+
 
 
 ## An example `fidesops.toml` configuration file
@@ -86,6 +89,9 @@ TASK_RETRY_DELAY=20
 TASK_RETRY_BACKOFF=2
 REQUIRE_MANUAL_REQUEST_APPROVAL=True
 MASKING_STRICT=True
+
+[user]
+ANALYTICS_OPT_OUT=True
 ```
 
 Please note: The configuration is case-sensitive, so the variables must be specified in UPPERCASE.

fidesops.toml

@@ -27,3 +27,6 @@ TASK_RETRY_DELAY=1
 TASK_RETRY_BACKOFF=1
 REQUIRE_MANUAL_REQUEST_APPROVAL=false
 MASKING_STRICT=true
+
+[user]
+ANALYTICS_OPT_OUT=true

requirements.txt

@@ -34,3 +34,4 @@ starlette~=0.14.2
 Unidecode==1.2.0
 uvicorn~=0.13.4
 versioneer==0.19
+fideslog==1.1.5

src/fidesops/analytics.py

@@ -0,0 +1,74 @@
+import logging
+from datetime import datetime, timezone
+from platform import system
+
+import versioneer
+from fideslog.sdk.python.client import AnalyticsClient
+from fideslog.sdk.python.event import AnalyticsEvent
+from fideslog.sdk.python.exceptions import AnalyticsError
+from fideslog.sdk.python.utils import generate_client_id
+
+from fidesops.core.config import config
+from fidesops.main import app
+from fidesops.schemas.analytics import FideslogAnalyticsEvent
+
+logger = logging.getLogger(__name__)
+
+
+def get_version() -> str:
+    """Version of Fidesops"""
+    return versioneer.get_version()
+
+
+def in_developer_mode() -> bool:
+    """True if runnning in dev mode"""
+    return config.dev_mode
+
+
+def in_docker_container() -> bool:
+    """`True` if the command was submitted within a Docker container. Default: `False`."""
+    return True
+
+
+def running_on_local_host() -> bool:
+    """For events submitted as a result of making API server requests, `True` if the API server is running on the user's local host. Default: `False`."""
+    return False
+
+
+class Analytics:
+    def __init__(self):
+        if not config.user.ANALYTICS_OPT_OUT:
+            client_id: str = generate_client_id(bytes(app.title, "utf-8"))
+            self.client = AnalyticsClient(
+                client_id=client_id,
+                developer_mode=in_developer_mode(),
+                extra_data={
+                    "this data": "will be included with every event sent by this client",
+                    "include": "any context that every event requires",
+                    "never include": "identifying information of any kind",
+                },
+                os=system(),
+                product_name=app.title,
+                production_version=get_version(),
+            )
+
+    def send_event(self, event_data: FideslogAnalyticsEvent):
+        if not config.user.ANALYTICS_OPT_OUT:
+            try:
+                cli_command_event = AnalyticsEvent(
+                    command=event_data.command,
+                    docker=in_docker_container(),
+                    event=event_data.event,
+                    error=event_data.error,
+                    event_created_at=datetime.now(tz=timezone.utc),
+                    extra_data=event_data.extra_data,
+                    flags=event_data.flags,
+                    local_host=running_on_local_host(),
+                    resource_counts=event_data.resource_counts,
+                    status_code=event_data.status_code,
+                )
+                self.client.send(cli_command_event)
+            except AnalyticsError as err:
+                logger.warning(f"Error sending analytics event: {err}")
+            else:
+                print(f"Analytics event sent: {event_data.event}")

src/fidesops/core/config.py

@@ -170,13 +170,20 @@ class Config:
         env_prefix = "FIDESOPS__SECURITY__"
 
 
+class UserSettings(FidesSettings):
+    """Configuration settings for Analytics variables."""
+
+    ANALYTICS_OPT_OUT: bool = True
+
+
 class FidesopsConfig(FidesSettings):
     """Configuration variables for the FastAPI project"""
 
     database: DatabaseSettings
     redis: RedisSettings
     security: SecuritySettings
     execution: ExecutionSettings
+    user: UserSettings
 
     is_test_mode: bool = os.getenv("TESTING") == "True"
     hot_reloading: bool = os.getenv("FIDESOPS__HOT_RELOAD") == "True"

src/fidesops/main.py

@@ -1,8 +1,10 @@
 import logging
+
 import uvicorn
 from fastapi import FastAPI
 from starlette.middleware.cors import CORSMiddleware
 
+from fidesops.analytics import Analytics
 from fidesops.api.v1.api import api_router
 from fidesops.api.v1.urn_registry import V1_URL_PREFIX
 from fidesops.db.database import init_db
@@ -29,6 +31,9 @@
 
 app.include_router(api_router)
 
+# fideslog
+Analytics()
+
 
 def start_webserver() -> None:
     """Run any pending DB migrations and start the webserver."""

src/fidesops/schemas/analytics.py

@@ -0,0 +1,25 @@
+from typing import Optional, List, Dict
+
+from fidesops.schemas.base_class import BaseSchema
+
+
+class FideslogAnalyticsEvent(BaseSchema):
+    """
+    Define a new analytics event to send to the fideslog server.
+    event: The name/type of this event.
+    command: For events submitted as a result of running CLI commands, the name of the command that was submitted. May include the subcommand name(s).
+    endpoint: For events submitted as a result of making API server requests, the API endpoint path that was requested. If a fully-qualified URL is provided, only the URL path will be persisted.
+    error: For events submitted as a result of running CLI commands that exit with a non-0 status code, or events submitted as a result of API server requests that respond with a non-2xx status code, the error type, without specific error details.
+    extra_data: Any additional key/value pairs that should be associated with this event.
+    flags: For events submitted as a result of running CLI commands, the flags in use when the command was submitted. Omits flag values (when they exist) by persisting only the portion of each string in this list that come before `=` or `space` characters.
+    resource_counts: Should contain the counts of dataset, policy, and system manifests in use when this event was submitted. Include all three keys, even if one or more of their values are `0`. Ex: `{ "datasets": 7, "policies": 26, "systems": 9 }`.
+    status_code: For events submitted as a result of making API server requests, the HTTP status code included in the response.
+    """
+    event: str
+    command: Optional[str] = None
+    endpoint: Optional[str] = None
+    error: Optional[str] = None
+    extra_data: Optional[Dict[str, str]] = None
+    flags: Optional[List[str]] = None
+    resource_counts: Optional[Dict[str, int]] = None
+    status_code: Optional[int] = None