Merge branch 'foodtruck-main' of deploy_amundsen:SaltIO/amundsen into…

… foodtruck-main

common/amundsen_common/models/table.py

@@ -69,6 +69,17 @@ class Meta:
         target = TypeMetadata
         register_as_scheme = True
 
+@attr.s(auto_attribs=True, kw_only=True)
+class ProgrammaticDescription:
+    source: str
+    text: str
+
+
+class ProgrammaticDescriptionSchema(AttrsSchema):
+    class Meta:
+        target = ProgrammaticDescription
+        register_as_scheme = True
+
 
 @attr.s(auto_attribs=True, kw_only=True)
 class Column:
@@ -80,6 +91,7 @@ class Column:
     stats: List[Stat] = []
     badges: Optional[List[Badge]] = []
     type_metadata: Optional[TypeMetadata] = None  # Used to support complex column types
+    programmatic_descriptions: List[ProgrammaticDescription] = []
 
 
 class ColumnSchema(AttrsSchema):
@@ -133,18 +145,6 @@ def default_if_none(arg: Optional[bool]) -> bool:
     return arg or False
 
 
-@attr.s(auto_attribs=True, kw_only=True)
-class ProgrammaticDescription:
-    source: str
-    text: str
-
-
-class ProgrammaticDescriptionSchema(AttrsSchema):
-    class Meta:
-        target = ProgrammaticDescription
-        register_as_scheme = True
-
-
 @attr.s(auto_attribs=True, kw_only=True)
 class TableSummary:
     database: str = attr.ib()

common/setup.py

@@ -4,7 +4,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '0.31.0+foodtruck.6'
+__version__ = '0.31.0+foodtruck.7'
 
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements-dev.txt')

databuilder/requirements.txt

@@ -27,7 +27,7 @@ responses>=0.10.6
 jsonref==0.2
 
 # amundsen-common>=0.16.0
-amundsen-common==0.31.0+foodtruck.6
+amundsen-common==0.31.0+foodtruck.7
 amundsen-rds==0.0.8
 
 queryparser-python3==0.7.0

databuilder/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '7.4.4+foodtruck.9'
+__version__ = '7.4.4+foodtruck.10'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  'requirements.txt')

frontend/amundsen_application/__init__.py

@@ -25,6 +25,7 @@
 from amundsen_application.api.quality.v0 import quality_blueprint
 from amundsen_application.api.search.v1 import search_blueprint
 from amundsen_application.api.notice.v0 import notices_blueprint
+from amundsen_application.api.file_upload.v0 import file_upload_blueprint
 from amundsen_application.api.v0 import blueprint
 from amundsen_application.deprecations import process_deprecations
 
@@ -94,6 +95,7 @@ def create_app(config_module_class: str = None, template_folder: str = None) ->
     app.register_blueprint(notices_blueprint)
     app.register_blueprint(api_bp)
     app.register_blueprint(dashboard_preview_blueprint)
+    app.register_blueprint(file_upload_blueprint)
     init_routes(app)
 
     init_custom_routes = app.config.get('INIT_CUSTOM_ROUTES')

frontend/amundsen_application/api/file_upload/__init__.py

@@ -0,0 +1,2 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0

frontend/amundsen_application/api/file_upload/v0.py

@@ -0,0 +1,115 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from pkg_resources import iter_entry_points
+
+from http import HTTPStatus
+
+import boto3
+from botocore.config import Config
+
+from flask import Response, jsonify, make_response, current_app as app, request
+from flask.blueprints import Blueprint
+from werkzeug.utils import import_string
+
+LOGGER = logging.getLogger(__name__)
+
+file_upload_blueprint = Blueprint('file_upload', __name__, url_prefix='/api/file_upload/v0')
+
+@file_upload_blueprint.route('/initiate_multipart_upload', methods=['POST'])
+def initiate_multipart_upload():
+
+    data = request.json
+    file_name = data.get('fileName')
+    file_type = data.get('fileType')
+
+    try:
+        # Initiate multipart upload
+        response = _get_s3_client().create_multipart_upload(
+            Bucket=_get_bucket_name(),
+            Key=_get_s3_file_key(file_name=file_name),
+            ContentType=file_type
+        )
+
+        return make_response(jsonify({'uploadId': response['UploadId']}), HTTPStatus.OK)
+    except Exception as e:
+        message = 'Encountered exception: ' + str(e)
+        LOGGER.exception(message)
+        payload = jsonify({'uploadId': '', 'msg': message})
+        return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
+@file_upload_blueprint.route('/get_presigned_url', methods=['POST'])
+def get_presigned_url() -> Response:
+    data = request.json
+    file_name = data.get('fileName')
+    upload_id = data.get('uploadId')
+    part_number = data.get('partNumber')  # The chunk/part number
+    content_type = data.get('contentType')
+
+
+    try:
+        # Generate presigned URL for the specific part
+        presigned_url = _get_s3_client().generate_presigned_url(
+            'upload_part',
+            Params={
+                'Bucket': _get_bucket_name(),
+                'Key': _get_s3_file_key(file_name=file_name),
+                'UploadId': upload_id,
+                'PartNumber': part_number,
+                # 'ContentType': content_type
+            },
+            ExpiresIn=3600  # 1 hour expiration
+        )
+
+        return make_response(jsonify({'url': presigned_url}), HTTPStatus.OK)
+
+    except Exception as e:
+        message = 'Encountered exception: ' + str(e)
+        LOGGER.exception(message)
+        payload = jsonify({'url': '', 'msg': message})
+        return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+@file_upload_blueprint.route('/complete_multipart_upload', methods=['POST'])
+def complete_multipart_upload():
+    data = request.json
+    file_name = data.get('fileName')
+    upload_id = data.get('uploadId')
+    parts = data.get('parts')  # List of parts {PartNumber, ETag}
+
+    try:
+        # Complete the multipart upload
+        response = _get_s3_client().complete_multipart_upload(
+            Bucket=_get_bucket_name(),
+            Key=_get_s3_file_key(file_name=file_name),
+            UploadId=upload_id,
+            MultipartUpload={
+                'Parts': parts
+            }
+        )
+
+        return make_response(jsonify(response), HTTPStatus.OK)
+
+    except Exception as e:
+        message = 'Encountered exception: ' + str(e)
+        LOGGER.exception(message)
+        payload = jsonify({'msg': message})
+        return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
+def _get_bucket_name() -> str:
+    return f'cmdrvl-metadata-{app.config["HOST_ID"]}'
+
+def _get_s3_client():
+    s3_client = boto3.client(
+        's3',
+        region_name=app.config['AWS_DEFAULT_REGION'],
+        aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'],
+        aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY'],
+        config=Config(signature_version='s3v4')
+    )
+    return s3_client
+
+def _get_s3_file_key(file_name: str) -> str:
+    return f'upload/{file_name}'

frontend/amundsen_application/api/utils/metadata_utils.py

@@ -197,6 +197,10 @@ def marshall_table_full(table_dict: Dict) -> Dict:
         col['key'] = results['key'] + '/' + col['name']
         # Set editable state
         col['is_editable'] = is_editable
+
+        prog_descriptions = col['programmatic_descriptions']
+        col['programmatic_descriptions'] = _convert_prog_descriptions(prog_descriptions)
+
         _recursive_set_type_metadata_is_editable(col['type_metadata'], is_editable)
         # If order is provided, we sort the column based on the pre-defined order
         if app.config['COLUMN_STAT_ORDER']:

frontend/amundsen_application/config.py

@@ -152,6 +152,11 @@ class Config:
     MTLS_CLIENT_KEY = os.getenv('MTLS_CLIENT_KEY')
     """Optional. The path to a PEM formatted key to use with the MTLS_CLIENT_CERT. MTLS_CLIENT_CERT must also be set."""
 
+    HOST_ID = os.getenv('HOST_ID')
+    AWS_DEFAULT_REGION = os.getenv('AWS_DEFAULT_REGION')
+    AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+    AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+
 
 class LocalConfig(Config):
     DEBUG = False

frontend/amundsen_application/static/js/components/EditableText/index.tsx

@@ -5,6 +5,10 @@ import * as autosize from 'autosize';
 import * as React from 'react';
 import * as ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
+import DOMPurify from 'dompurify';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { dark } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import he from 'he';
 
 import LoadingSpinnerOverlay from 'components/LoadingSpinnerOverlay';
 import { EditableSectionChildProps } from 'components/EditableSection';
@@ -21,10 +25,8 @@ import {
 } from './constants';
 
 import './styles.scss';
-import { getGPTResponse } from 'ducks/ai/reducer';
-import { GetGPTResponse, GetGPTResponseRequest, GetGPTResponseResponse } from 'ducks/ai/types';
+import { GetGPTResponseRequest, GetGPTResponseResponse } from 'ducks/ai/types';
 import { GPTResponse } from 'interfaces/AI';
-import { PROPOSITION_LABEL } from 'features/Feedback/constants';
 
 export interface StateFromProps {
   refreshValue?: string;
@@ -212,6 +214,20 @@ class EditableText extends React.Component<
     }
   };
 
+  isJsonString = (str: string) => {
+    let isJson = false;
+    try {
+      const parsed = JSON.parse(he.decode(str));
+      isJson = typeof parsed === 'object' && parsed !== null;
+    } catch (e) {
+      console.log(e)
+      isJson = false;
+    }
+
+    console.log(`isJson=${isJson}`)
+    return isJson
+  };
+
   render() {
     const { isEditing, editable, maxLength, allowDangerousHtml } = this.props;
     const { value = '', isDisabled, isAIEnabled, isGPTResponseLoading, aiError } = this.state;
@@ -223,16 +239,29 @@ class EditableText extends React.Component<
     }
 
     if (!isEditing) {
+      const sanitizedContent = allowDangerousHtml ? DOMPurify.sanitize(value) : value;
+      const isJson = this.isJsonString(sanitizedContent)
+
       return (
         <div className="editable-text">
-          <div className="markdown-wrapper">
-            <ReactMarkdown
-              allowDangerousHtml={!!allowDangerousHtml}
-              plugins={[remarkGfm]}
-            >
-              {value}
-            </ReactMarkdown>
-          </div>
+            <div className="markdown-wrapper">
+              {isJson ?
+                (
+                  <div style={{ width: '400px', height: '500px', overflow: 'auto', border: '1px solid #ddd' }}>
+                    <SyntaxHighlighter language="json" style={dark}>
+                      {JSON.stringify(JSON.parse(he.decode(sanitizedContent)), null, 2)}
+                    </SyntaxHighlighter>
+                  </div>
+                ) : (
+                  <ReactMarkdown
+                    allowDangerousHtml={!!allowDangerousHtml}
+                    plugins={[remarkGfm]}
+                  >
+                    {sanitizedContent}
+                  </ReactMarkdown>
+                )
+              }
+            </div>
           {editable && !value && (
             <button
               className="edit-link btn btn-link"