353- initial infra for fideslog integration (#541)

* 353- initial infra for fideslog integration

* adds changelog

* cr and adds storage of analytics id in config

* implement internal mode, exclude tests/CI, implement sending server start event

* remove validationerr

* format

* sort

* lint

* cr changes

* lint fixes

* Adds root_user to test toml config

* Add analytics opt out arg to parser of run_infrastructure

* implicit true when passing env var to docker-compose

* small code style changes, and updating test fidesops.toml to opt out of analytics by default

* missing comma

* spacing issue

* remove fidesctl dep

CHANGELOG.md

@@ -106,6 +106,7 @@ The types of changes are:
 * Frontend for privacy request denial reaons [#480](https://github.com/ethyca/fidesops/pull/480)
 * Publish Fidesops to Pypi [#491](https://github.com/ethyca/fidesops/pull/491)
 * DRP data rights endpoint [#526](https://github.com/ethyca/fidesops/pull/526)
+* ADDS Fideslog integration [#541](https://github.com/ethyca/fidesops/pull/541)
 
 
 ### Changed

Dockerfile

@@ -43,4 +43,7 @@ COPY . /fidesops
 WORKDIR /fidesops
 RUN pip install -e .
 
+# Enable detection of running within Docker
+ENV RUNNING_IN_DOCKER=true
+
 CMD [ "fidesops", "webserver" ]

Makefile

@@ -28,7 +28,7 @@ help:
 
 init-db: compose-build
 	@echo "Check for new migrations to run..."
-	@docker-compose run --rm $(IMAGE_NAME) \
+	@docker-compose run --rm -e ANALYTICS_OPT_OUT $(IMAGE_NAME) \
 	python -c "\
 	from fidesops.db.database import init_db; \
 	from fidesops.core.config import config; \
@@ -84,14 +84,16 @@ check-all: isort-ci black-ci pylint mypy check-migrations pytest pytest-integrat
 
 black-ci: compose-build
 	@echo "Running black checks..."
-	@docker-compose run $(IMAGE_NAME) \
+	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
+		$(IMAGE_NAME) \
 		black --check src/ tests/ \
 		|| (echo "Error running 'black --check src/ tests/', please run 'make black' to format your code!"; exit 1)
 	@make teardown
 
 check-migrations: compose-build
 	@echo "Check if there are unrun migrations..."
-	@docker-compose run --rm $(IMAGE_NAME) \
+	@docker-compose run --rm -e ANALYTICS_OPT_OUT $(IMAGE_NAME) \
 	python -c "\
 	from fidesops.db.database import check_missing_migrations; \
 	from fidesops.core.config import config; \
@@ -105,32 +107,39 @@ isort-ci:
 
 pylint: compose-build
 	@echo "Running pylint checks..."
-	@docker-compose run $(IMAGE_NAME) \
+	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
+		$(IMAGE_NAME) \
 		pylint src/
 	@make teardown
 
 mypy: compose-build
 	@echo "Running mypy checks..."
-	@docker-compose run $(IMAGE_NAME) \
+	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
+		$(IMAGE_NAME) \
 		mypy src/
 	@make teardown
 
 pytest: compose-build
 	@echo "Running pytest unit tests..."
-	@docker-compose run $(IMAGE_NAME) \
+	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
+		$(IMAGE_NAME) \
 		pytest $(pytestpath) -m "not integration and not integration_external and not integration_saas"
 	@make teardown
 
 pytest-integration:
 	@virtualenv -p python3 fidesops_test_dispatch; \
 		source fidesops_test_dispatch/bin/activate; \
-		python run_infrastructure.py --run_tests --datastores $(datastores)
+		python run_infrastructure.py --run_tests --analytics_opt_out --datastores $(datastores)
 	@make teardown
 
 # These tests connect to external third-party test databases
 pytest-integration-external: compose-build
 	@echo "Running tests that connect to external third party test databases"
 	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
 		-e REDSHIFT_TEST_URI \
 		-e SNOWFLAKE_TEST_URI -e REDSHIFT_TEST_DB_SCHEMA \
 		-e BIGQUERY_KEYFILE_CREDS -e BIGQUERY_DATASET \
@@ -140,6 +149,7 @@ pytest-integration-external: compose-build
 pytest-saas: compose-build
 	@echo "Running integration tests for SaaS connectors"
 	@docker-compose run \
+		-e ANALYTICS_OPT_OUT \
 		-e MAILCHIMP_DOMAIN -e MAILCHIMP_USERNAME -e MAILCHIMP_API_KEY -e MAILCHIMP_IDENTITY_EMAIL \
 		-e SENTRY_HOST -e SENTRY_ACCESS_TOKEN -e SENTRY_IDENTITY_EMAIL -e SENTRY_ERASURE_TOKEN -e SENTRY_ERASURE_IDENTITY -e SENTRY_USER_ID -e SENTRY_ISSUE_URL  \
 		-e HUBSPOT_DOMAIN -e HUBSPOT_HAPIKEY -e HUBSPOT_IDENTITY_EMAIL \

data/config/fidesops.toml

@@ -1,34 +1,37 @@
-# Configuration TOML used in select unit tests
-PORT=8080
+PORT = 8080
 
 [database]
-SERVER="testserver"
-USER="testuser"
-PASSWORD="testpassword"
-DB="testdb"
-TEST_DB="test_testdb"
-ENABLED=true
+SERVER = "testserver"
+USER = "testuser"
+PASSWORD = "testpassword"
+DB = "testdb"
+TEST_DB = "test_testdb"
+ENABLED = true
 
 [redis]
-HOST="testredis"
-PASSWORD="testpassword"
-PORT=1234
-CHARSET="utf8"
-DEFAULT_TTL_SECONDS=1000
-DB_INDEX=0
-ENABLED=true
+HOST = "testredis"
+PASSWORD = "testpassword"
+PORT = 1234
+CHARSET = "utf8"
+DEFAULT_TTL_SECONDS = 1000
+DB_INDEX = 0
+ENABLED = true
 
 [security]
-APP_ENCRYPTION_KEY="atestencryptionkeythatisvalidlen"
-CORS_ORIGINS=["http://test.com", "https://test.com"]
-OAUTH_ROOT_CLIENT_ID="testrootclientid"
-OAUTH_ROOT_CLIENT_SECRET="testrootclientsecret"
-DRP_JWT_SECRET="testdrpsecret"
+APP_ENCRYPTION_KEY = "atestencryptionkeythatisvalidlen"
+CORS_ORIGINS = [ "http://test.com", "https://test.com",]
+OAUTH_ROOT_CLIENT_ID = "testrootclientid"
+OAUTH_ROOT_CLIENT_SECRET = "testrootclientsecret"
+DRP_JWT_SECRET = "testdrpsecret"
 LOG_LEVEL = "DEBUG"
 
 [execution]
-TASK_RETRY_COUNT=0
-TASK_RETRY_DELAY=1
-TASK_RETRY_BACKOFF=1
-REQUIRE_MANUAL_REQUEST_APPROVAL=false
-MASKING_STRICT=true
+TASK_RETRY_COUNT = 0
+TASK_RETRY_DELAY = 1
+TASK_RETRY_BACKOFF = 1
+REQUIRE_MANUAL_REQUEST_APPROVAL = false
+MASKING_STRICT = true
+
+[root_user]
+ANALYTICS_OPT_OUT = true
+ANALYTICS_ID = "internal"

docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - FIDESOPS__DEV_MODE=${FIDESOPS__DEV_MODE}
       - FIDESOPS__LOG_PII=${FIDESOPS__LOG_PII}
       - FIDESOPS__HOT_RELOAD=${FIDESOPS__HOT_RELOAD}
+      - FIDESOPS__ROOT_USER__ANALYTICS_ID=${FIDESOPS__ROOT_USER__ANALYTICS_ID}
 
   db:
     image: postgres:12

docs/fidesops/docs/guides/configuration_reference.md

@@ -56,6 +56,9 @@ The `fidesops.toml` file should specify the following variables:
 |`TASK_RETRY_BACKOFF` | `FIDESOPS__EXECUTION__TASK_RETRY_BACKOFF` | int | 2 | 1 | The backoff factor for retries, to space out repeated retries.
 |`REQUIRE_MANUAL_REQUEST_APPROVAL` | `FIDESOPS__EXECUTION__REQUIRE_MANUAL_REQUEST_APPROVAL` | bool | False | False | Whether privacy requests require explicit approval to execute
 |`MASKING_STRICT` | `FIDESOPS__EXECUTION__MASKING_STRICT` | bool | True | True | If MASKING_STRICT is True, we only use "update" requests to mask data. (For third-party integrations, you should define an `update` endpoint to use.)  If MASKING_STRICT is False, you are allowing fidesops to use any defined DELETE or GDPR DELETE endpoints to remove PII. In this case, you should define `delete` or `data_protection_request` endpoints for your third-party integrations.  Note that setting MASKING_STRICT to False means that data may be deleted beyond the specific data categories that you've configured in your Policy.
+|---|---|---|---|---|---|
+|`ANALYTICS_OPT_OUT` | `FIDESOPS__USER__ANALYTICS_OPT_OUT` | bool | False | False | Opt out of sending anonymous usage data to Ethyca to improve the product experience
+
 
 
 ## An example `fidesops.toml` configuration file
@@ -91,6 +94,9 @@ TASK_RETRY_DELAY=20
 TASK_RETRY_BACKOFF=2
 REQUIRE_MANUAL_REQUEST_APPROVAL=true
 MASKING_STRICT=true
+
+[root_user]
+ANALYTICS_OPT_OUT=false
 ```
 
 Please note: The configuration is case-sensitive, so the variables must be specified in UPPERCASE.

fidesops.toml

@@ -1,34 +1,37 @@
 PORT=8080
 
 [database]
-SERVER="db"
-USER="postgres"
-PASSWORD="216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a"
-DB="app"
-TEST_DB="test"
-ENABLED=true
+SERVER = "db"
+USER = "postgres"
+PASSWORD = "216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a"
+DB = "app"
+TEST_DB = "test"
+ENABLED = true
 
 [redis]
-HOST="redis"
-PASSWORD="testpassword"
-PORT=6379
-CHARSET="utf8"
-DEFAULT_TTL_SECONDS=604800
-DB_INDEX=0
-ENABLED=true
+HOST = "redis"
+PASSWORD = "testpassword"
+PORT = 6379
+CHARSET = "utf8"
+DEFAULT_TTL_SECONDS = 604800
+DB_INDEX = 0
+ENABLED = true
 
 [security]
-APP_ENCRYPTION_KEY="OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
-CORS_ORIGINS=["http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001"]
-ENCODING="UTF-8"
-OAUTH_ROOT_CLIENT_ID="fidesopsadmin"
-OAUTH_ROOT_CLIENT_SECRET="fidesopsadminsecret"
-DRP_JWT_SECRET="secret"
+APP_ENCRYPTION_KEY = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
+CORS_ORIGINS = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",]
+ENCODING = "UTF-8"
+OAUTH_ROOT_CLIENT_ID = "fidesopsadmin"
+OAUTH_ROOT_CLIENT_SECRET = "fidesopsadminsecret"
+DRP_JWT_SECRET = "secret"
 LOG_LEVEL = "INFO"
 
 [execution]
-TASK_RETRY_COUNT=0
-TASK_RETRY_DELAY=1
-TASK_RETRY_BACKOFF=1
-REQUIRE_MANUAL_REQUEST_APPROVAL=false
-MASKING_STRICT=true
+TASK_RETRY_COUNT = 0
+TASK_RETRY_DELAY = 1
+TASK_RETRY_BACKOFF = 1
+REQUIRE_MANUAL_REQUEST_APPROVAL = false
+MASKING_STRICT = true
+
+[root_user]
+ANALYTICS_OPT_OUT = false

requirements.txt

@@ -11,6 +11,7 @@ fastapi-caching[redis]
 fastapi-pagination[sqlalchemy]~= 0.8.3
 fastapi[all]==0.68.1
 fideslang==1.0.0
+fideslog==1.1.5
 multidimensional_urlencode==0.0.4
 pandas==1.3.3
 passlib[bcrypt]==1.7.2

run_infrastructure.py

@@ -36,6 +36,7 @@ def run_infrastructure(
     run_tests: bool = False,  # Should we run the tests after creating the infra?
     run_create_superuser: bool = False,  # Should we run the create_superuser command?
     run_create_test_data: bool = False,  # Should we run the create_test_data command?
+    analytics_opt_out: bool = False,  # Should we opt out of analytics?
 ) -> None:
     """
     - Create a Docker Compose file path for all datastores specified in `datastores`.
@@ -91,6 +92,7 @@ def run_infrastructure(
             datastores,
             docker_compose_path=path,
             pytest_path=pytest_path,
+            analytics_opt_out=analytics_opt_out,
         )
 
     elif run_create_superuser:
@@ -205,6 +207,7 @@ def _run_tests(
     datastores: List[str],
     docker_compose_path: str,
     pytest_path: str = "",
+    analytics_opt_out: bool = False,
 ) -> None:
     """
     Runs unit tests against the specified datastores
@@ -233,6 +236,9 @@ def _run_tests(
             for env_var in EXTERNAL_DATASTORE_CONFIG[datastore]:
                 environment_variables += f"-e {env_var} "
 
+    if analytics_opt_out:
+        environment_variables += "-e ANALYTICS_OPT_OUT"
+
     pytest_path += f' -m "{pytest_markers}"'
 
     _run_cmd_or_err(
@@ -295,6 +301,11 @@ def _run_tests(
         "--run_create_test_data",
         action="store_true",
     )
+    parser.add_argument(
+        "-a",
+        "--analytics_opt_out",
+        action="store_true",
+    )
 
     config_args = parser.parse_args()
 
@@ -307,4 +318,5 @@ def _run_tests(
         run_tests=config_args.run_tests,
         run_create_superuser=config_args.run_create_superuser,
         run_create_test_data=config_args.run_create_test_data,
+        analytics_opt_out=config_args.analytics_opt_out,
     )

src/fidesops/analytics.py

@@ -0,0 +1,45 @@
+import logging
+import os
+from platform import system
+
+from fideslog.sdk.python.client import AnalyticsClient
+from fideslog.sdk.python.event import AnalyticsEvent
+from fideslog.sdk.python.exceptions import AnalyticsError
+
+from fidesops import __version__ as fidesops_version
+from fidesops.core.config import config
+
+logger = logging.getLogger(__name__)
+
+
+def in_docker_container() -> bool:
+    """`True` if the command was submitted within a Docker container. Default: `False`."""
+    return bool(os.getenv("RUNNING_IN_DOCKER") == "true")
+
+
+def running_on_local_host() -> bool:
+    """For events submitted as a result of making API server requests, `True` if the API server is running on the user's local host. Default: `False`."""
+    return True
+
+
+analytics_client = AnalyticsClient(
+    client_id=config.root_user.ANALYTICS_ID,
+    developer_mode=config.dev_mode,
+    extra_data=None,
+    os=system(),
+    product_name="fidesops",
+    production_version=fidesops_version,
+)
+
+
+def send_analytics_event(event: AnalyticsEvent) -> None:
+    if config.root_user.ANALYTICS_OPT_OUT:
+        return
+    try:
+        analytics_client.send(event)
+    except AnalyticsError as err:
+        logger.warning(f"Error sending analytics event: {err}")
+    else:
+        logger.info(
+            f"Analytics event sent: {event.event} with client id: {analytics_client.client_id}"
+        )