diff --git a/common/amundsen_common/models/table.py b/common/amundsen_common/models/table.py
index 81c8c6d245..d57e103402 100644
--- a/common/amundsen_common/models/table.py
+++ b/common/amundsen_common/models/table.py
@@ -69,6 +69,17 @@ class Meta:
target = TypeMetadata
register_as_scheme = True
+@attr.s(auto_attribs=True, kw_only=True)
+class ProgrammaticDescription:
+ source: str
+ text: str
+
+
+class ProgrammaticDescriptionSchema(AttrsSchema):
+ class Meta:
+ target = ProgrammaticDescription
+ register_as_scheme = True
+
@attr.s(auto_attribs=True, kw_only=True)
class Column:
@@ -80,6 +91,7 @@ class Column:
stats: List[Stat] = []
badges: Optional[List[Badge]] = []
type_metadata: Optional[TypeMetadata] = None # Used to support complex column types
+ programmatic_descriptions: List[ProgrammaticDescription] = []
class ColumnSchema(AttrsSchema):
@@ -133,18 +145,6 @@ def default_if_none(arg: Optional[bool]) -> bool:
return arg or False
-@attr.s(auto_attribs=True, kw_only=True)
-class ProgrammaticDescription:
- source: str
- text: str
-
-
-class ProgrammaticDescriptionSchema(AttrsSchema):
- class Meta:
- target = ProgrammaticDescription
- register_as_scheme = True
-
-
@attr.s(auto_attribs=True, kw_only=True)
class TableSummary:
database: str = attr.ib()
diff --git a/common/setup.py b/common/setup.py
index 3b10c1bb89..cd2b834ef6 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -4,7 +4,7 @@
from setuptools import find_packages, setup
-__version__ = '0.31.0+foodtruck.6'
+__version__ = '0.31.0+foodtruck.7'
requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements-dev.txt')
diff --git a/databuilder/requirements.txt b/databuilder/requirements.txt
index bd4f857957..de2afd2242 100644
--- a/databuilder/requirements.txt
+++ b/databuilder/requirements.txt
@@ -27,7 +27,7 @@ responses>=0.10.6
jsonref==0.2
# amundsen-common>=0.16.0
-amundsen-common==0.31.0+foodtruck.6
+amundsen-common==0.31.0+foodtruck.7
amundsen-rds==0.0.8
queryparser-python3==0.7.0
diff --git a/databuilder/setup.py b/databuilder/setup.py
index 802df78b6f..7af5d047d7 100644
--- a/databuilder/setup.py
+++ b/databuilder/setup.py
@@ -5,7 +5,7 @@
from setuptools import find_packages, setup
-__version__ = '7.4.4+foodtruck.9'
+__version__ = '7.4.4+foodtruck.10'
requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'requirements.txt')
diff --git a/frontend/amundsen_application/__init__.py b/frontend/amundsen_application/__init__.py
index 87d44942e9..914dc762aa 100644
--- a/frontend/amundsen_application/__init__.py
+++ b/frontend/amundsen_application/__init__.py
@@ -25,6 +25,7 @@
from amundsen_application.api.quality.v0 import quality_blueprint
from amundsen_application.api.search.v1 import search_blueprint
from amundsen_application.api.notice.v0 import notices_blueprint
+from amundsen_application.api.file_upload.v0 import file_upload_blueprint
from amundsen_application.api.v0 import blueprint
from amundsen_application.deprecations import process_deprecations
@@ -94,6 +95,7 @@ def create_app(config_module_class: str = None, template_folder: str = None) ->
app.register_blueprint(notices_blueprint)
app.register_blueprint(api_bp)
app.register_blueprint(dashboard_preview_blueprint)
+ app.register_blueprint(file_upload_blueprint)
init_routes(app)
init_custom_routes = app.config.get('INIT_CUSTOM_ROUTES')
diff --git a/frontend/amundsen_application/api/file_upload/__init__.py b/frontend/amundsen_application/api/file_upload/__init__.py
new file mode 100644
index 0000000000..f3145d75b3
--- /dev/null
+++ b/frontend/amundsen_application/api/file_upload/__init__.py
@@ -0,0 +1,2 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/frontend/amundsen_application/api/file_upload/v0.py b/frontend/amundsen_application/api/file_upload/v0.py
new file mode 100644
index 0000000000..6a2b04f4e7
--- /dev/null
+++ b/frontend/amundsen_application/api/file_upload/v0.py
@@ -0,0 +1,115 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from pkg_resources import iter_entry_points
+
+from http import HTTPStatus
+
+import boto3
+from botocore.config import Config
+
+from flask import Response, jsonify, make_response, current_app as app, request
+from flask.blueprints import Blueprint
+from werkzeug.utils import import_string
+
+LOGGER = logging.getLogger(__name__)
+
+file_upload_blueprint = Blueprint('file_upload', __name__, url_prefix='/api/file_upload/v0')
+
+@file_upload_blueprint.route('/initiate_multipart_upload', methods=['POST'])
+def initiate_multipart_upload():
+
+ data = request.json
+ file_name = data.get('fileName')
+ file_type = data.get('fileType')
+
+ try:
+ # Initiate multipart upload
+ response = _get_s3_client().create_multipart_upload(
+ Bucket=_get_bucket_name(),
+ Key=_get_s3_file_key(file_name=file_name),
+ ContentType=file_type
+ )
+
+ return make_response(jsonify({'uploadId': response['UploadId']}), HTTPStatus.OK)
+ except Exception as e:
+ message = 'Encountered exception: ' + str(e)
+ LOGGER.exception(message)
+ payload = jsonify({'uploadId': '', 'msg': message})
+ return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
+@file_upload_blueprint.route('/get_presigned_url', methods=['POST'])
+def get_presigned_url() -> Response:
+ data = request.json
+ file_name = data.get('fileName')
+ upload_id = data.get('uploadId')
+ part_number = data.get('partNumber') # The chunk/part number
+ content_type = data.get('contentType')
+
+
+ try:
+ # Generate presigned URL for the specific part
+ presigned_url = _get_s3_client().generate_presigned_url(
+ 'upload_part',
+ Params={
+ 'Bucket': _get_bucket_name(),
+ 'Key': _get_s3_file_key(file_name=file_name),
+ 'UploadId': upload_id,
+ 'PartNumber': part_number,
+ # 'ContentType': content_type
+ },
+ ExpiresIn=3600 # 1 hour expiration
+ )
+
+ return make_response(jsonify({'url': presigned_url}), HTTPStatus.OK)
+
+ except Exception as e:
+ message = 'Encountered exception: ' + str(e)
+ LOGGER.exception(message)
+ payload = jsonify({'url': '', 'msg': message})
+ return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+@file_upload_blueprint.route('/complete_multipart_upload', methods=['POST'])
+def complete_multipart_upload():
+ data = request.json
+ file_name = data.get('fileName')
+ upload_id = data.get('uploadId')
+ parts = data.get('parts') # List of parts {PartNumber, ETag}
+
+ try:
+ # Complete the multipart upload
+ response = _get_s3_client().complete_multipart_upload(
+ Bucket=_get_bucket_name(),
+ Key=_get_s3_file_key(file_name=file_name),
+ UploadId=upload_id,
+ MultipartUpload={
+ 'Parts': parts
+ }
+ )
+
+ return make_response(jsonify(response), HTTPStatus.OK)
+
+ except Exception as e:
+ message = 'Encountered exception: ' + str(e)
+ LOGGER.exception(message)
+ payload = jsonify({'msg': message})
+ return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
+def _get_bucket_name() -> str:
+ return f'cmdrvl-metadata-{app.config["HOST_ID"]}'
+
+def _get_s3_client():
+ s3_client = boto3.client(
+ 's3',
+ region_name=app.config['AWS_DEFAULT_REGION'],
+ aws_access_key_id=app.config['AWS_ACCESS_KEY_ID'],
+ aws_secret_access_key=app.config['AWS_SECRET_ACCESS_KEY'],
+ config=Config(signature_version='s3v4')
+ )
+ return s3_client
+
+def _get_s3_file_key(file_name: str) -> str:
+ return f'upload/{file_name}'
\ No newline at end of file
diff --git a/frontend/amundsen_application/api/utils/metadata_utils.py b/frontend/amundsen_application/api/utils/metadata_utils.py
index 7f1f017988..60da1f59eb 100644
--- a/frontend/amundsen_application/api/utils/metadata_utils.py
+++ b/frontend/amundsen_application/api/utils/metadata_utils.py
@@ -197,6 +197,10 @@ def marshall_table_full(table_dict: Dict) -> Dict:
col['key'] = results['key'] + '/' + col['name']
# Set editable state
col['is_editable'] = is_editable
+
+ prog_descriptions = col['programmatic_descriptions']
+ col['programmatic_descriptions'] = _convert_prog_descriptions(prog_descriptions)
+
_recursive_set_type_metadata_is_editable(col['type_metadata'], is_editable)
# If order is provided, we sort the column based on the pre-defined order
if app.config['COLUMN_STAT_ORDER']:
diff --git a/frontend/amundsen_application/config.py b/frontend/amundsen_application/config.py
index 1b2ae5ddd8..c08d027e71 100644
--- a/frontend/amundsen_application/config.py
+++ b/frontend/amundsen_application/config.py
@@ -152,6 +152,11 @@ class Config:
MTLS_CLIENT_KEY = os.getenv('MTLS_CLIENT_KEY')
"""Optional. The path to a PEM formatted key to use with the MTLS_CLIENT_CERT. MTLS_CLIENT_CERT must also be set."""
+ HOST_ID = os.getenv('HOST_ID')
+ AWS_DEFAULT_REGION = os.getenv('AWS_DEFAULT_REGION')
+ AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+ AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+
class LocalConfig(Config):
DEBUG = False
diff --git a/frontend/amundsen_application/static/js/components/EditableText/index.tsx b/frontend/amundsen_application/static/js/components/EditableText/index.tsx
index cf20ffc9ce..e1ab2c7ff2 100644
--- a/frontend/amundsen_application/static/js/components/EditableText/index.tsx
+++ b/frontend/amundsen_application/static/js/components/EditableText/index.tsx
@@ -5,6 +5,10 @@ import * as autosize from 'autosize';
import * as React from 'react';
import * as ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
+import DOMPurify from 'dompurify';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { dark } from 'react-syntax-highlighter/dist/esm/styles/prism';
+import he from 'he';
import LoadingSpinnerOverlay from 'components/LoadingSpinnerOverlay';
import { EditableSectionChildProps } from 'components/EditableSection';
@@ -21,10 +25,8 @@ import {
} from './constants';
import './styles.scss';
-import { getGPTResponse } from 'ducks/ai/reducer';
-import { GetGPTResponse, GetGPTResponseRequest, GetGPTResponseResponse } from 'ducks/ai/types';
+import { GetGPTResponseRequest, GetGPTResponseResponse } from 'ducks/ai/types';
import { GPTResponse } from 'interfaces/AI';
-import { PROPOSITION_LABEL } from 'features/Feedback/constants';
export interface StateFromProps {
refreshValue?: string;
@@ -212,6 +214,20 @@ class EditableText extends React.Component<
}
};
+ isJsonString = (str: string) => {
+ let isJson = false;
+ try {
+ const parsed = JSON.parse(he.decode(str));
+ isJson = typeof parsed === 'object' && parsed !== null;
+ } catch (e) {
+ console.log(e)
+ isJson = false;
+ }
+
+ console.log(`isJson=${isJson}`)
+ return isJson
+ };
+
render() {
const { isEditing, editable, maxLength, allowDangerousHtml } = this.props;
const { value = '', isDisabled, isAIEnabled, isGPTResponseLoading, aiError } = this.state;
@@ -223,16 +239,29 @@ class EditableText extends React.Component<
}
if (!isEditing) {
+ const sanitizedContent = allowDangerousHtml ? DOMPurify.sanitize(value) : value;
+ const isJson = this.isJsonString(sanitizedContent)
+
return (
-
-
- {value}
-
-
+
+ {isJson ?
+ (
+
+
+ {JSON.stringify(JSON.parse(he.decode(sanitizedContent)), null, 2)}
+
+
+ ) : (
+
+ {sanitizedContent}
+
+ )
+ }
+
{editable && !value && (