Use tailwind theme colors and improve dark mode colors

backend/.gitignore

@@ -8,4 +8,4 @@ api_keys.py
 .env
 vespa-app.zip
 dynamic_config_storage/
-celerybeat-schedule
+celerybeat-schedule*

backend/Dockerfile

@@ -42,6 +42,15 @@ RUN apt-get remove -y --allow-remove-essential perl-base xserver-common xvfb cma
     rm -rf /var/lib/apt/lists/* && \
     rm /usr/local/lib/python3.11/site-packages/tornado/test/test.key
 
+# Pre-downloading models for setups with limited egress
+RUN python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('intfloat/e5-base-v2')"
+
+# Pre-downloading NLTK for setups with limited egress
+RUN python -c "import nltk; \
+nltk.download('stopwords', quiet=True); \
+nltk.download('wordnet', quiet=True); \
+nltk.download('punkt', quiet=True);"
+
 # Set up application files
 WORKDIR /app
 COPY ./danswer /app/danswer

backend/Dockerfile.model_server

@@ -17,6 +17,16 @@ RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt
 RUN apt-get remove -y --allow-remove-essential perl-base && \
     apt-get autoremove -y
 
+# Pre-downloading models for setups with limited egress
+RUN python -c "from transformers import AutoModel, AutoTokenizer, TFDistilBertForSequenceClassification; \
+from huggingface_hub import snapshot_download; \
+AutoTokenizer.from_pretrained('danswer/intent-model'); \
+AutoTokenizer.from_pretrained('intfloat/e5-base-v2'); \
+AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
+snapshot_download('danswer/intent-model'); \
+snapshot_download('intfloat/e5-base-v2'); \
+snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1')"
+
 WORKDIR /app
 
 # Utils used by model server

backend/alembic/versions/401c1ac29467_add_tables_for_ui_based_llm_.py

@@ -0,0 +1,49 @@
+"""Add tables for UI-based LLM configuration
+
+Revision ID: 401c1ac29467
+Revises: 703313b75876
+Create Date: 2024-04-13 18:07:29.153817
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "401c1ac29467"
+down_revision = "703313b75876"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "llm_provider",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("api_key", sa.String(), nullable=True),
+        sa.Column("api_base", sa.String(), nullable=True),
+        sa.Column("api_version", sa.String(), nullable=True),
+        sa.Column(
+            "custom_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        sa.Column("default_model_name", sa.String(), nullable=False),
+        sa.Column("fast_default_model_name", sa.String(), nullable=True),
+        sa.Column("is_default_provider", sa.Boolean(), unique=True, nullable=True),
+        sa.Column("model_names", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+
+    op.add_column(
+        "persona",
+        sa.Column("llm_model_provider_override", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "llm_model_provider_override")
+
+    op.drop_table("llm_provider")

backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py

@@ -0,0 +1,83 @@
+"""Add TokenRateLimit Tables
+
+Revision ID: 703313b75876
+Revises: fad14119fb92
+Create Date: 2024-04-15 01:36:02.952809
+
+"""
+import json
+from typing import cast
+from alembic import op
+import sqlalchemy as sa
+from danswer.dynamic_configs.factory import get_dynamic_config_store
+
+# revision identifiers, used by Alembic.
+revision = "703313b75876"
+down_revision = "fad14119fb92"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "token_rate_limit",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("token_budget", sa.Integer(), nullable=False),
+        sa.Column("period_hours", sa.Integer(), nullable=False),
+        sa.Column(
+            "scope",
+            sa.String(length=10),
+            nullable=False,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "token_rate_limit__user_group",
+        sa.Column("rate_limit_id", sa.Integer(), nullable=False),
+        sa.Column("user_group_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["rate_limit_id"],
+            ["token_rate_limit.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.PrimaryKeyConstraint("rate_limit_id", "user_group_id"),
+    )
+
+    try:
+        settings_json = cast(
+            str, get_dynamic_config_store().load("token_budget_settings")
+        )
+        settings = json.loads(settings_json)
+
+        is_enabled = settings.get("enable_token_budget", False)
+        token_budget = settings.get("token_budget", -1)
+        period_hours = settings.get("period_hours", -1)
+
+        if is_enabled and token_budget > 0 and period_hours > 0:
+            op.execute(
+                f"INSERT INTO token_rate_limit \
+                    (enabled, token_budget, period_hours, scope) VALUES \
+                        ({is_enabled}, {token_budget}, {period_hours}, 'GLOBAL')"
+            )
+
+        # Delete the dynamic config
+        get_dynamic_config_store().delete("token_budget_settings")
+
+    except Exception:
+        # Ignore if the dynamic config is not found
+        pass
+
+
+def downgrade() -> None:
+    op.drop_table("token_rate_limit__user_group")
+    op.drop_table("token_rate_limit")

backend/alembic/versions/72bdc9929a46_permission_auto_sync_framework.py

@@ -0,0 +1,81 @@
+"""Permission Auto Sync Framework
+
+Revision ID: 72bdc9929a46
+Revises: 475fcefe8826
+Create Date: 2024-04-14 21:15:28.659634
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "72bdc9929a46"
+down_revision = "475fcefe8826"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "email_to_external_user_cache",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("external_user_id", sa.String(), nullable=False),
+        sa.Column("user_id", sa.UUID(), nullable=True),
+        sa.Column("user_email", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "external_permission",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.UUID(), nullable=True),
+        sa.Column("user_email", sa.String(), nullable=False),
+        sa.Column(
+            "source_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("external_permission_group", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "permission_sync_run",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "source_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("update_type", sa.String(), nullable=False),
+        sa.Column("cc_pair_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "status",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("error_msg", sa.Text(), nullable=True),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["cc_pair_id"],
+            ["connector_credential_pair.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("permission_sync_run")
+    op.drop_table("external_permission")
+    op.drop_table("email_to_external_user_cache")

backend/alembic/versions/ef7da92f7213_add_files_to_chatmessage.py

@@ -0,0 +1,27 @@
+"""Add files to ChatMessage
+
+Revision ID: ef7da92f7213
+Revises: 401c1ac29467
+Create Date: 2024-04-28 16:59:33.199153
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "ef7da92f7213"
+down_revision = "401c1ac29467"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_message",
+        sa.Column("files", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_message", "files")

backend/alembic/versions/fad14119fb92_delete_tags_with_wrong_enum.py

@@ -0,0 +1,39 @@
+"""Delete Tags with wrong Enum
+
+Revision ID: fad14119fb92
+Revises: 72bdc9929a46
+Create Date: 2024-04-25 17:05:09.695703
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "fad14119fb92"
+down_revision = "72bdc9929a46"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Some documents may lose their tags but this is the only way as the enum
+    # mapping may have changed since tag switched to string (it will be reindexed anyway)
+    op.execute(
+        """
+        DELETE FROM document__tag
+        WHERE tag_id IN (
+            SELECT id FROM tag
+            WHERE source ~ '^[0-9]+$'
+        )
+        """
+    )
+
+    op.execute(
+        """
+        DELETE FROM tag
+        WHERE source ~ '^[0-9]+$'
+        """
+    )
+
+
+def downgrade() -> None:
+    pass

backend/danswer/auth/users.py

@@ -42,6 +42,9 @@
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import AuthType
+from danswer.configs.constants import DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN
+from danswer.configs.constants import DANSWER_API_KEY_PREFIX
+from danswer.configs.constants import UNNAMED_KEY_PLACEHOLDER
 from danswer.db.auth import get_access_token_db
 from danswer.db.auth import get_user_count
 from danswer.db.auth import get_user_db
@@ -69,6 +72,20 @@ def verify_auth_setting() -> None:
     logger.info(f"Using Auth Type: {AUTH_TYPE.value}")
 
 
+def get_display_email(email: str | None, space_less: bool = False) -> str:
+    if email and email.endswith(DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN):
+        name = email.split("@")[0]
+        if name == DANSWER_API_KEY_PREFIX + UNNAMED_KEY_PLACEHOLDER:
+            return "Unnamed API Key"
+
+        if space_less:
+            return name
+
+        return name.replace("API_KEY__", "API Key: ")
+
+    return email or ""
+
+
 def user_needs_to_be_verified() -> bool:
     # all other auth types besides basic should require users to be
     # verified