From 2b89f4a40700e96ffc10a6e1766e90d296cc73de Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 16 Dec 2020 22:02:44 -0500 Subject: [PATCH 01/11] First pass implementation of graphql schema in server for metadata --- lib/pbench/server/api/__init__.py | 13 ++- .../server/api/resources/graphql_api.py | 83 ++++++++++++++++++- .../server/api/resources/graphql_schema.py | 74 +++++++++++++++++ 3 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 lib/pbench/server/api/resources/graphql_schema.py diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 8657c57c96..666aafd062 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -7,14 +7,14 @@ import os import sys -from flask import Flask +from flask import Flask, request from flask_restful import Api from flask_cors import CORS from pbench.server import PbenchServerConfig from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified from pbench.server.api.resources.upload_api import Upload, HostInfo -from pbench.server.api.resources.graphql_api import GraphQL +from pbench.server.api.resources.graphql_api import GraphQL, UserMetadata from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch from pbench.server.api.resources.query_apis.query_controllers import QueryControllers @@ -84,6 +84,10 @@ def register_endpoints(api, app, config): resource_class_args=(logger, token_auth), ) + api.add_resource( + UserMetadata, f"{base_uri}/user/metadata", resource_class_args=(config, logger), + ) + def get_server_config(): cfg_name = os.environ.get("_PBENCH_SERVER_CONFIG") @@ -121,6 +125,11 @@ def create_app(server_config): app.logger.exception("Exception while initializing sqlalchemy database") sys.exit(1) + @app.before_request + def before_request(): + print(request.path) + print(request.remote_addr) + @app.teardown_appcontext def shutdown_session(exception=None): Database.db_session.remove() diff --git a/lib/pbench/server/api/resources/graphql_api.py b/lib/pbench/server/api/resources/graphql_api.py index 2f6e41a861..343a363df5 100644 --- a/lib/pbench/server/api/resources/graphql_api.py +++ b/lib/pbench/server/api/resources/graphql_api.py @@ -1,6 +1,87 @@ import requests from flask_restful import Resource, abort -from flask import request, make_response +from flask import request, make_response, jsonify +from pbench.server.api.resources.auth import auth +from pbench.server.api.resources.graphql_schema import schema + + +class UserMetadata(Resource): + """ + Abstracted pbench API for handling user metadata by using graphql schema + """ + + def __init__(self, config, logger): + self.server_config = config + self.logger = logger + + @auth.login_required() + def post(self): + """ + Post request for creating metadata instance for a user. + This requires a JWT auth token in the header field + + This requires a JSON data with required user metadata fields + { + "config": "config", + "description": "description", + } + + Required headers include + Authorization: JWT token (user received upon login) + + :return: JSON Payload + response_object = { + "status": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: %s", request.url) + abort(400, message="Invalid json object in request") + + config = post_data.get("config") + if not config: + self.logger.warning("Config not provided during metadata creation") + abort(400, message="Please provide a config string") + + description = post_data.get("description") + if not description: + self.logger.warning("Description not provided during metadata creation") + abort(400, message="Please provide a description string") + current_user_id = auth.current_user().id + try: + # query GraphQL + query = f""" + mutation {{ + createMetadata (input: {{config:"{config}", description:"{description}", user_id:{current_user_id}}}) {{ + metadata {{ + id + config + description + }} + }} + }} + """ + result = schema.execute(query) + except Exception as e: + self.logger.exception("Exception occurred during Metadata creation") + abort(500, message="INTERNAL ERROR") + else: + data = result.data["createMetadata"]["metadata"] + response_object = { + "status": "success", + "data": { + "id": data["id"], + "config": data["config"], + "description": data["description"], + }, + } + return make_response(jsonify(response_object), 201) class GraphQL(Resource): diff --git a/lib/pbench/server/api/resources/graphql_schema.py b/lib/pbench/server/api/resources/graphql_schema.py new file mode 100644 index 0000000000..f8fbe0c6eb --- /dev/null +++ b/lib/pbench/server/api/resources/graphql_schema.py @@ -0,0 +1,74 @@ +from pbench.server.api.resources.models import MetadataModel +from pbench.server.api.resources.database import Database + +import graphene +from graphene import relay +from graphene_sqlalchemy import SQLAlchemyObjectType + + +# Define graphql types +class Metadata(SQLAlchemyObjectType): + class Meta: + model = MetadataModel + interfaces = (relay.Node,) + + +class MetadataAttribute: + id = graphene.ID + user_id = graphene.ID() + created = graphene.DateTime() + updated = graphene.DateTime() + config = graphene.String() + description = graphene.String() + + +class CreateMetadataInput(graphene.InputObjectType, MetadataAttribute): + pass + + +# mutations +class CreateMetadata(graphene.Mutation): + metadata = graphene.Field(lambda: Metadata) + ok = graphene.Boolean() + + class Arguments: + input = CreateMetadataInput(required=True) + + @staticmethod + def mutate(self, info, input): + data = input + metadata = MetadataModel(**data) + Database.db_session.add(metadata) + Database.db_session.commit() + ok = True + return CreateMetadata(metadata=metadata, ok=ok) + + +class Mutation(graphene.ObjectType): + createMetadata = CreateMetadata.Field() + + +# Query +class Query(graphene.ObjectType): + node = relay.Node.Field() + + metadata_by_id = graphene.List(Metadata, id=graphene.String()) + metadata_by_userid = graphene.List(Metadata, userid=graphene.String()) + + @staticmethod + def resolve_metadata_by_id(parent, info, **args): + q = args.get("id") + + metadata_query = Metadata.get_query(info) + return metadata_query.filter(MetadataModel.id == q).all() + + @staticmethod + def resolve_metadata_by_userid(parent, info, **args): + q = args.get("userid") + + metadata_query = Metadata.get_query(info) + return metadata_query.filter(MetadataModel.user_id == q).all() + + +# schema +schema = graphene.Schema(query=Query, mutation=Mutation, types=[Metadata]) From d3f1a4b129dc216bc9d02cf2b3fed79997c149e4 Mon Sep 17 00:00:00 2001 From: npalaska Date: Thu, 14 Jan 2021 14:52:49 -0500 Subject: [PATCH 02/11] Add Metadata API support --- lib/pbench/server/api/__init__.py | 5 +- .../server/api/resources/graphql_api.py | 205 ++++++++++++++++-- 2 files changed, 190 insertions(+), 20 deletions(-) diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 666aafd062..f968351d62 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -14,7 +14,7 @@ from pbench.server import PbenchServerConfig from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified from pbench.server.api.resources.upload_api import Upload, HostInfo -from pbench.server.api.resources.graphql_api import GraphQL, UserMetadata +from pbench.server.api.resources.graphql_api import GraphQL, UserMetadata, QueryMetadata from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch from pbench.server.api.resources.query_apis.query_controllers import QueryControllers @@ -87,6 +87,9 @@ def register_endpoints(api, app, config): api.add_resource( UserMetadata, f"{base_uri}/user/metadata", resource_class_args=(config, logger), ) + api.add_resource( + QueryMetadata, f"{base_uri}/user/metadata/", resource_class_args=(config, logger), + ) def get_server_config(): diff --git a/lib/pbench/server/api/resources/graphql_api.py b/lib/pbench/server/api/resources/graphql_api.py index 343a363df5..05f8f77cf3 100644 --- a/lib/pbench/server/api/resources/graphql_api.py +++ b/lib/pbench/server/api/resources/graphql_api.py @@ -1,8 +1,11 @@ import requests +import datetime +import json from flask_restful import Resource, abort from flask import request, make_response, jsonify from pbench.server.api.resources.auth import auth -from pbench.server.api.resources.graphql_schema import schema +from pbench.server.api.resources.models import MetadataModel +from pbench.server.api.resources.database import Database class UserMetadata(Resource): @@ -54,35 +57,199 @@ def post(self): self.logger.warning("Description not provided during metadata creation") abort(400, message="Please provide a description string") current_user_id = auth.current_user().id + try: - # query GraphQL - query = f""" - mutation {{ - createMetadata (input: {{config:"{config}", description:"{description}", user_id:{current_user_id}}}) {{ - metadata {{ - id - config - description - }} - }} - }} - """ - result = schema.execute(query) - except Exception as e: + # Create a new metadata session + metadata_session = MetadataModel( + created=str(datetime.datetime.now()), + config=config, + description=description, + user_id=current_user_id + ) + # insert the metadata session for a user + Database.db_session.add(metadata_session) + Database.db_session.commit() + self.logger.info("New user metadata session created") + except Exception: self.logger.exception("Exception occurred during Metadata creation") abort(500, message="INTERNAL ERROR") else: - data = result.data["createMetadata"]["metadata"] response_object = { "status": "success", "data": { - "id": data["id"], - "config": data["config"], - "description": data["description"], + "id": metadata_session.id, + "config": metadata_session.config, + "description": metadata_session.description, }, } return make_response(jsonify(response_object), 201) + @auth.login_required() + def get(self): + """ + Get request for querying all the metadata sessions for a user. + This requires a JWT auth token in the header field + + Required headers include + Authorization: JWT token (user received upon login) + + :return: JSON Payload + response_object = { + "status": "success" + "data": { + "sessions": [ + {"id": "metadata_id", + "config": "Config string" + "description": "Description string"}, + {}] + } + } + """ + current_user_id = auth.current_user().id + try: + # Fetch the metadata session + sessions = ( + Database.db_session.query(MetadataModel) + .filter_by(user_id=current_user_id) + .all() + ) + + req_keys = ["id", "config", "description", "created"] + data = json.dumps([{key: session.as_dict()[key] for key in req_keys} for session in sessions]) + except Exception: + self.logger.exception("Exception occurred during querying Metadata model") + abort(500, message="INTERNAL ERROR") + + response_object = { + "status": "success", + "data": { + "sessions": data + }, + } + return make_response(jsonify(response_object), 200) + + +class QueryMetadata(Resource): + """ + Abstracted pbench API for querying a single user metadata session + """ + + def __init__(self, config, logger): + self.server_config = config + self.logger = logger + + @auth.login_required() + def get(self, id=None): + """ + Get request for querying a metadata session for a user given a metadata id. + This requires a JWT auth token in the header field + + This requires a JSON data with required user metadata fields to update + { + "description": "description", + } + + The url requires a metadata session id such as /user/metadata/ + + Required headers include + Authorization: JWT token (user received upon login) + + :return: JSON Payload + response_object = { + "status": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + if not id: + self.logger.warning("Meatadata id not provided during metadata query") + abort(400, message="Please provide a metadata id to query") + + try: + # Fetch the metadata session + session = ( + Database.db_session.query(MetadataModel) + .filter_by(id=id) + .first() + ) + except Exception: + self.logger.exception("Exception occurred during querying Metadata model") + abort(500, message="INTERNAL ERROR") + else: + response_object = { + "status": "success", + "data": { + "id": session.id, + "config": session.config, + "description": session.description, + }, + } + return make_response(jsonify(response_object), 200) + + @auth.login_required() + def put(self, id=None): + """ + Put request for updating a metadata session for a user given a metadata id. + This requires a JWT auth token in the header field + + The url requires a metadata session id such as /user/metadata/ + + Required headers include + Authorization: JWT token (user received upon login) + + :return: JSON Payload + response_object = { + "status": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + if not id: + self.logger.warning("Meatadata id not provided during metadata query") + abort(400, message="Please provide a metadata id to query") + + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: %s", request.url) + abort(400, message="Invalid json object in request") + + description = post_data.get("description") + if not description: + self.logger.warning("Description not provided during metadata update") + abort(400, message="Please provide a description string") + + try: + # Fetch the metadata session + session = ( + Database.db_session.query(MetadataModel) + .filter_by(id=id) + .first() + ) + session.description = description + # Update the metadata session for a user + Database.db_session.add(session) + Database.db_session.commit() + self.logger.info("User metadata session updated") + except Exception: + self.logger.exception("Exception occurred during querying Metadata model") + abort(500, message="INTERNAL ERROR") + else: + response_object = { + "status": "success", + "data": { + "id": session.id, + "config": session.config, + "description": session.description, + }, + } + return make_response(jsonify(response_object), 200) + class GraphQL(Resource): """GraphQL API for post request via server.""" From e0d5cd11280af05aec6aad13047bc32c8af8e3f2 Mon Sep 17 00:00:00 2001 From: npalaska Date: Thu, 14 Jan 2021 14:58:07 -0500 Subject: [PATCH 03/11] Add unit test for metadata creation and querying --- lib/pbench/test/unit/server/test_user_auth.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py index 7f6923c0b0..ae9a9b7cff 100644 --- a/lib/pbench/test/unit/server/test_user_auth.py +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -3,6 +3,7 @@ from pbench.server.database.models.users import User from pbench.server.database.models.active_tokens import ActiveTokens from pbench.server.database.database import Database +from pbench.server.api.resources.models import MetadataModel def register_user( @@ -423,3 +424,50 @@ def test_delete_user(client, server_config): data = response.json assert data["message"] == "Successfully deleted." assert response.status_code == 200 + + +class TestMetadataSession: + @staticmethod + def test_registration(client, server_config, pytestconfig): + client.config["SESSION_FILE_DIR"] = pytestconfig.cache.get("TMP", None) + """ Test for user registration """ + with client: + response = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data = response.json + assert data["status"] == "success" + + response = login_user(client, server_config, "user", "12345") + data_login = response.json + assert data_login["status"] == "success" + + response = client.post( + f"{server_config.rest_uri}/user/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]) + ) + data = response.json + assert data["status"] == "success" + + response = client.post( + f"{server_config.rest_uri}/user/metadata", + json={"config": "config2", "description": "description2"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]) + ) + data = response.json + assert data["status"] == "success" + + response = client.get( + f"{server_config.rest_uri}/user/metadata", + headers=dict(Authorization="Bearer " + data_login["auth_token"]) + ) + data = response.json + assert data["status"] == "success" + assert response.status_code == 200 \ No newline at end of file From 4bd78a48073dc1e9e4b21834d5d63481f700bba0 Mon Sep 17 00:00:00 2001 From: npalaska Date: Tue, 16 Feb 2021 19:18:20 -0500 Subject: [PATCH 04/11] rebasing and some fixes --- lib/pbench/server/api/__init__.py | 9 ++---- .../server/api/resources/graphql_api.py | 21 +++++++------ lib/pbench/server/database/models/metadata.py | 31 +++++++++++++++++++ lib/pbench/test/unit/server/test_user_auth.py | 10 +++--- 4 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 lib/pbench/server/database/models/metadata.py diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index f968351d62..895df57b7f 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -85,10 +85,10 @@ def register_endpoints(api, app, config): ) api.add_resource( - UserMetadata, f"{base_uri}/user/metadata", resource_class_args=(config, logger), + UserMetadata, f"{base_uri}/metadata", resource_class_args=(config, logger, token_auth), ) api.add_resource( - QueryMetadata, f"{base_uri}/user/metadata/", resource_class_args=(config, logger), + QueryMetadata, f"{base_uri}/metadata/", resource_class_args=(config, logger), ) @@ -128,11 +128,6 @@ def create_app(server_config): app.logger.exception("Exception while initializing sqlalchemy database") sys.exit(1) - @app.before_request - def before_request(): - print(request.path) - print(request.remote_addr) - @app.teardown_appcontext def shutdown_session(exception=None): Database.db_session.remove() diff --git a/lib/pbench/server/api/resources/graphql_api.py b/lib/pbench/server/api/resources/graphql_api.py index 05f8f77cf3..29f64101cb 100644 --- a/lib/pbench/server/api/resources/graphql_api.py +++ b/lib/pbench/server/api/resources/graphql_api.py @@ -3,9 +3,9 @@ import json from flask_restful import Resource, abort from flask import request, make_response, jsonify -from pbench.server.api.resources.auth import auth -from pbench.server.api.resources.models import MetadataModel -from pbench.server.api.resources.database import Database +from pbench.server.database.models.metadata import MetadataModel +from pbench.server.database.database import Database +from pbench.server.api.auth import Auth class UserMetadata(Resource): @@ -13,11 +13,12 @@ class UserMetadata(Resource): Abstracted pbench API for handling user metadata by using graphql schema """ - def __init__(self, config, logger): + def __init__(self, config, logger, auth): self.server_config = config self.logger = logger + self.auth = auth - @auth.login_required() + @Auth.token_auth.login_required(f=Auth().verify_auth()) def post(self): """ Post request for creating metadata instance for a user. @@ -56,7 +57,7 @@ def post(self): if not description: self.logger.warning("Description not provided during metadata creation") abort(400, message="Please provide a description string") - current_user_id = auth.current_user().id + current_user_id = self.auth.token_auth.current_user().id try: # Create a new metadata session @@ -84,7 +85,7 @@ def post(self): } return make_response(jsonify(response_object), 201) - @auth.login_required() + @Auth.token_auth.login_required(f=Auth().verify_auth()) def get(self): """ Get request for querying all the metadata sessions for a user. @@ -105,7 +106,7 @@ def get(self): } } """ - current_user_id = auth.current_user().id + current_user_id = self.auth.token_auth.current_user().id try: # Fetch the metadata session sessions = ( @@ -138,7 +139,7 @@ def __init__(self, config, logger): self.server_config = config self.logger = logger - @auth.login_required() + @Auth.token_auth.login_required(f=Auth().verify_auth()) def get(self, id=None): """ Get request for querying a metadata session for a user given a metadata id. @@ -189,7 +190,7 @@ def get(self, id=None): } return make_response(jsonify(response_object), 200) - @auth.login_required() + @Auth.token_auth.login_required(f=Auth().verify_auth()) def put(self, id=None): """ Put request for updating a metadata session for a user given a metadata id. diff --git a/lib/pbench/server/database/models/metadata.py b/lib/pbench/server/database/models/metadata.py new file mode 100644 index 0000000000..69385d980d --- /dev/null +++ b/lib/pbench/server/database/models/metadata.py @@ -0,0 +1,31 @@ +import datetime +from dateutil import parser +from pbench.server.database.database import Database +from sqlalchemy import Column, Integer, String, DateTime, ForeignKey + + +class MetadataModel(Database.Base): + """ Metadata Model for storing user metadata details """ + + # TODO: Think about the better name + __tablename__ = "metadata" + + id = Column(Integer, primary_key=True, autoincrement=True) + created = Column(DateTime, nullable=False) + updated = Column(DateTime, nullable=False) + config = Column(String(255), unique=False, nullable=False) + description = Column(String(255), nullable=False) + user_id = Column(Integer, ForeignKey('users.id')) + + def __init__(self, created, config, description, user_id): + self.created = parser.parse(created) + self.updated = datetime.datetime.now() + self.config = config + self.description = description + self.user_id = user_id + + def __str__(self): + return f"Url id: {self.id}, created on: {self.created}, description: {self.description}" + + def as_dict(self): + return {c.name: str(getattr(self, c.name)) for c in self.__table__.columns} \ No newline at end of file diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py index ae9a9b7cff..a1c2632998 100644 --- a/lib/pbench/test/unit/server/test_user_auth.py +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -428,8 +428,7 @@ def test_delete_user(client, server_config): class TestMetadataSession: @staticmethod - def test_registration(client, server_config, pytestconfig): - client.config["SESSION_FILE_DIR"] = pytestconfig.cache.get("TMP", None) + def test_registration(client, server_config): """ Test for user registration """ with client: response = register_user( @@ -447,9 +446,10 @@ def test_registration(client, server_config, pytestconfig): response = login_user(client, server_config, "user", "12345") data_login = response.json assert data_login["status"] == "success" + assert data_login["auth_token"] response = client.post( - f"{server_config.rest_uri}/user/metadata", + f"{server_config.rest_uri}/metadata", json={"config": "config1", "description": "description1"}, headers=dict(Authorization="Bearer " + data_login["auth_token"]) ) @@ -457,7 +457,7 @@ def test_registration(client, server_config, pytestconfig): assert data["status"] == "success" response = client.post( - f"{server_config.rest_uri}/user/metadata", + f"{server_config.rest_uri}/metadata", json={"config": "config2", "description": "description2"}, headers=dict(Authorization="Bearer " + data_login["auth_token"]) ) @@ -465,7 +465,7 @@ def test_registration(client, server_config, pytestconfig): assert data["status"] == "success" response = client.get( - f"{server_config.rest_uri}/user/metadata", + f"{server_config.rest_uri}/metadata", headers=dict(Authorization="Bearer " + data_login["auth_token"]) ) data = response.json From 4c16e71477857894485456be18dc4721e9b8bcce Mon Sep 17 00:00:00 2001 From: npalaska Date: Mon, 22 Feb 2021 08:50:18 -0500 Subject: [PATCH 05/11] fix naming convention --- lib/pbench/test/unit/server/test_user_auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py index a1c2632998..4e2ab6f7b1 100644 --- a/lib/pbench/test/unit/server/test_user_auth.py +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -3,7 +3,6 @@ from pbench.server.database.models.users import User from pbench.server.database.models.active_tokens import ActiveTokens from pbench.server.database.database import Database -from pbench.server.api.resources.models import MetadataModel def register_user( @@ -428,7 +427,7 @@ def test_delete_user(client, server_config): class TestMetadataSession: @staticmethod - def test_registration(client, server_config): + def test_metadata_creation(client, server_config): """ Test for user registration """ with client: response = register_user( From a32ad0b0361d6f9387c4cff508abc6d8ad0bbe89 Mon Sep 17 00:00:00 2001 From: npalaska Date: Tue, 9 Mar 2021 00:30:48 -0500 Subject: [PATCH 06/11] Refactoring, address comments, more unit tests, metadata_api updates --- lib/pbench/server/api/__init__.py | 13 +- .../server/api/resources/graphql_api.py | 251 +------------ .../server/api/resources/metadata_api.py | 331 ++++++++++++++++++ lib/pbench/server/database/models/metadata.py | 86 ++++- .../unit/server/test_metadata_sessions.py | 223 ++++++++++++ lib/pbench/test/unit/server/test_user_auth.py | 47 --- 6 files changed, 634 insertions(+), 317 deletions(-) create mode 100644 lib/pbench/server/api/resources/metadata_api.py create mode 100644 lib/pbench/test/unit/server/test_metadata_sessions.py diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 895df57b7f..5e5d3603db 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -7,14 +7,15 @@ import os import sys -from flask import Flask, request +from flask import Flask from flask_restful import Api from flask_cors import CORS from pbench.server import PbenchServerConfig from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified from pbench.server.api.resources.upload_api import Upload, HostInfo -from pbench.server.api.resources.graphql_api import GraphQL, UserMetadata, QueryMetadata +from pbench.server.api.resources.graphql_api import GraphQL +from pbench.server.api.resources.metadata_api import UserMetadata, QueryMetadata from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch from pbench.server.api.resources.query_apis.query_controllers import QueryControllers @@ -85,10 +86,14 @@ def register_endpoints(api, app, config): ) api.add_resource( - UserMetadata, f"{base_uri}/metadata", resource_class_args=(config, logger, token_auth), + UserMetadata, + f"{base_uri}/metadata", + resource_class_args=(config, logger, token_auth), ) api.add_resource( - QueryMetadata, f"{base_uri}/metadata/", resource_class_args=(config, logger), + QueryMetadata, + f"{base_uri}/metadata/", + resource_class_args=(config, logger), ) diff --git a/lib/pbench/server/api/resources/graphql_api.py b/lib/pbench/server/api/resources/graphql_api.py index 29f64101cb..2f6e41a861 100644 --- a/lib/pbench/server/api/resources/graphql_api.py +++ b/lib/pbench/server/api/resources/graphql_api.py @@ -1,255 +1,6 @@ import requests -import datetime -import json from flask_restful import Resource, abort -from flask import request, make_response, jsonify -from pbench.server.database.models.metadata import MetadataModel -from pbench.server.database.database import Database -from pbench.server.api.auth import Auth - - -class UserMetadata(Resource): - """ - Abstracted pbench API for handling user metadata by using graphql schema - """ - - def __init__(self, config, logger, auth): - self.server_config = config - self.logger = logger - self.auth = auth - - @Auth.token_auth.login_required(f=Auth().verify_auth()) - def post(self): - """ - Post request for creating metadata instance for a user. - This requires a JWT auth token in the header field - - This requires a JSON data with required user metadata fields - { - "config": "config", - "description": "description", - } - - Required headers include - Authorization: JWT token (user received upon login) - - :return: JSON Payload - response_object = { - "status": "success" - "data" { - "id": "metadata_id", - "config": "Config string" - "description": "Description string" - } - } - """ - post_data = request.get_json() - if not post_data: - self.logger.warning("Invalid json object: %s", request.url) - abort(400, message="Invalid json object in request") - - config = post_data.get("config") - if not config: - self.logger.warning("Config not provided during metadata creation") - abort(400, message="Please provide a config string") - - description = post_data.get("description") - if not description: - self.logger.warning("Description not provided during metadata creation") - abort(400, message="Please provide a description string") - current_user_id = self.auth.token_auth.current_user().id - - try: - # Create a new metadata session - metadata_session = MetadataModel( - created=str(datetime.datetime.now()), - config=config, - description=description, - user_id=current_user_id - ) - # insert the metadata session for a user - Database.db_session.add(metadata_session) - Database.db_session.commit() - self.logger.info("New user metadata session created") - except Exception: - self.logger.exception("Exception occurred during Metadata creation") - abort(500, message="INTERNAL ERROR") - else: - response_object = { - "status": "success", - "data": { - "id": metadata_session.id, - "config": metadata_session.config, - "description": metadata_session.description, - }, - } - return make_response(jsonify(response_object), 201) - - @Auth.token_auth.login_required(f=Auth().verify_auth()) - def get(self): - """ - Get request for querying all the metadata sessions for a user. - This requires a JWT auth token in the header field - - Required headers include - Authorization: JWT token (user received upon login) - - :return: JSON Payload - response_object = { - "status": "success" - "data": { - "sessions": [ - {"id": "metadata_id", - "config": "Config string" - "description": "Description string"}, - {}] - } - } - """ - current_user_id = self.auth.token_auth.current_user().id - try: - # Fetch the metadata session - sessions = ( - Database.db_session.query(MetadataModel) - .filter_by(user_id=current_user_id) - .all() - ) - - req_keys = ["id", "config", "description", "created"] - data = json.dumps([{key: session.as_dict()[key] for key in req_keys} for session in sessions]) - except Exception: - self.logger.exception("Exception occurred during querying Metadata model") - abort(500, message="INTERNAL ERROR") - - response_object = { - "status": "success", - "data": { - "sessions": data - }, - } - return make_response(jsonify(response_object), 200) - - -class QueryMetadata(Resource): - """ - Abstracted pbench API for querying a single user metadata session - """ - - def __init__(self, config, logger): - self.server_config = config - self.logger = logger - - @Auth.token_auth.login_required(f=Auth().verify_auth()) - def get(self, id=None): - """ - Get request for querying a metadata session for a user given a metadata id. - This requires a JWT auth token in the header field - - This requires a JSON data with required user metadata fields to update - { - "description": "description", - } - - The url requires a metadata session id such as /user/metadata/ - - Required headers include - Authorization: JWT token (user received upon login) - - :return: JSON Payload - response_object = { - "status": "success" - "data" { - "id": "metadata_id", - "config": "Config string" - "description": "Description string" - } - } - """ - if not id: - self.logger.warning("Meatadata id not provided during metadata query") - abort(400, message="Please provide a metadata id to query") - - try: - # Fetch the metadata session - session = ( - Database.db_session.query(MetadataModel) - .filter_by(id=id) - .first() - ) - except Exception: - self.logger.exception("Exception occurred during querying Metadata model") - abort(500, message="INTERNAL ERROR") - else: - response_object = { - "status": "success", - "data": { - "id": session.id, - "config": session.config, - "description": session.description, - }, - } - return make_response(jsonify(response_object), 200) - - @Auth.token_auth.login_required(f=Auth().verify_auth()) - def put(self, id=None): - """ - Put request for updating a metadata session for a user given a metadata id. - This requires a JWT auth token in the header field - - The url requires a metadata session id such as /user/metadata/ - - Required headers include - Authorization: JWT token (user received upon login) - - :return: JSON Payload - response_object = { - "status": "success" - "data" { - "id": "metadata_id", - "config": "Config string" - "description": "Description string" - } - } - """ - if not id: - self.logger.warning("Meatadata id not provided during metadata query") - abort(400, message="Please provide a metadata id to query") - - post_data = request.get_json() - if not post_data: - self.logger.warning("Invalid json object: %s", request.url) - abort(400, message="Invalid json object in request") - - description = post_data.get("description") - if not description: - self.logger.warning("Description not provided during metadata update") - abort(400, message="Please provide a description string") - - try: - # Fetch the metadata session - session = ( - Database.db_session.query(MetadataModel) - .filter_by(id=id) - .first() - ) - session.description = description - # Update the metadata session for a user - Database.db_session.add(session) - Database.db_session.commit() - self.logger.info("User metadata session updated") - except Exception: - self.logger.exception("Exception occurred during querying Metadata model") - abort(500, message="INTERNAL ERROR") - else: - response_object = { - "status": "success", - "data": { - "id": session.id, - "config": session.config, - "description": session.description, - }, - } - return make_response(jsonify(response_object), 200) +from flask import request, make_response class GraphQL(Resource): diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py new file mode 100644 index 0000000000..8305cdeeb5 --- /dev/null +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -0,0 +1,331 @@ +from flask_restful import Resource, abort +from flask import request, make_response, jsonify +from pbench.server.database.models.metadata import Metadata +from pbench.server.api.auth import Auth + + +class UserMetadata(Resource): + """ + Abstracted pbench API for handling user metadata by using graphql schema + """ + + def __init__(self, config, logger, auth): + self.server_config = config + self.logger = logger + self.auth = auth + + @Auth.token_auth.login_required() + def post(self): + """ + Post request for creating metadata instance for a user. + This requires a Pbench auth token in the header field + + This requires a JSON data with required user metadata fields + { + "config": "config", + "description": "description", + } + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: {}", request.url) + abort(400, message="Invalid json object in request") + + current_user_id = self.auth.token_auth.current_user().id + + config = post_data.get("config") + if not config: + self.logger.warning( + "Config not provided during metadata creation. user_id: {}", + current_user_id, + ) + abort(400, message="Config field missing") + + description = post_data.get("description") + if not description: + self.logger.warning( + "Description not provided during metadata creation by user: {}", + current_user_id, + ) + abort(400, message="Description field missing") + + try: + # Create a new metadata session + metadata_session = Metadata( + config=config, description=description, user_id=current_user_id + ) + # insert the metadata session for a user + metadata_session.add() + self.logger.info( + "New metadata session created for user_id {}", current_user_id + ) + except Exception: + self.logger.exception("Exception occurred during the Metadata creation") + abort(500, message="INTERNAL ERROR") + else: + response_object = { + "message": "success", + "data": { + "id": metadata_session.id, + "config": metadata_session.config, + "description": metadata_session.description, + }, + } + return make_response(jsonify(response_object), 201) + + @Auth.token_auth.login_required() + def get(self): + """ + Get request for querying all the metadata sessions for a user. + returns the list of all the metadata sessions associated with a logged in user. + This requires a Pbench auth token in the header field + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "status": "success" + "data": { + "sessions": [ + {"id": "metadata_id", + "config": "Config string" + "description": "Description string"}, + {}] + } + } + """ + current_user_id = self.auth.token_auth.current_user().id + try: + # Fetch the metadata session + metadata_sessions = Metadata.query(user_id=current_user_id) + data = [session.get_json() for session in metadata_sessions] + except Exception: + self.logger.exception( + "Exception occurred while querying the Metadata model" + ) + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "success", + "data": {"sessions": data}, + } + return make_response(jsonify(response_object), 200) + + +class QueryMetadata(Resource): + """ + Abstracted pbench API for querying a single user metadata session + """ + + def __init__(self, config, logger): + self.server_config = config + self.logger = logger + + def verify_metadata(self, session, action): + current_user_id = Auth.token_auth.current_user().id + metadata_user_id = session.user_id + if current_user_id != metadata_user_id: + self.logger.warning( + "Metadata {}: Logged in user_id {} is different than the one provided in the URI {}", + action, + current_user_id, + metadata_user_id, + ) + abort(403, message="Not authorized to perform the specified action") + + @Auth.token_auth.login_required() + def get(self, id=None): + """ + Get request for querying a metadata session of a user given a metadata id. + This requires a Pbench auth token in the header field + + The url requires a metadata session id such as /user/metadata/ + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + if not id: + self.logger.warning("Metadata id not provided during metadata query") + abort(400, message="Please provide a metadata id to query") + + try: + # Fetch the metadata session + metadata_session = Metadata.query(id=id) + except Exception: + self.logger.exception( + "Exception occurred in the GET request while querying the Metadata model, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + # Verify if the metadata session id in the url belongs to the logged in user + self.verify_metadata(metadata_session, "get") + + response_object = { + "message": "success", + "data": { + "id": metadata_session.id, + "config": metadata_session.config, + "description": metadata_session.description, + }, + } + return make_response(jsonify(response_object), 200) + + @Auth.token_auth.login_required() + def put(self, id=None): + """ + Put request for updating a metadata session of a user given a metadata id. + This requires a Pbench auth token in the header field + + The url requires a metadata session id such as /user/metadata/ + + This requires a JSON data with required user metadata fields to update + { + "description": "description", + } + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "success" + "data" { + "id": "metadata_id", + "config": "Config string" + "description": "Description string" + } + } + """ + if not id: + self.logger.warning("Metadata id not provided during metadata query") + abort(400, message="Please provide a metadata id to query") + + post_data = request.get_json() + if not post_data: + self.logger.warning("Invalid json object: {}", request.url) + abort(400, message="Invalid json object in request") + + try: + metadata_session = Metadata.query(id=id) + except Exception: + self.logger.exception( + "Exception occurred in the PUT request while querying the Metadata model, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + # Verify if the metadata session id in the url belongs to the logged in user + self.verify_metadata(metadata_session, "put") + + # Check if the metadata payload contain fields that are either protected or + # not present in the metadata db. If any key in the payload does not match + # with the column name we will abort the update request. + non_existent = set(post_data.keys()).difference( + set(Metadata.__table__.columns.keys()) + ) + if non_existent: + self.logger.warning( + "User trying to update fields that are not present in the metadata database. Fields: {}", + non_existent, + ) + abort(400, message="Invalid fields in update request payload") + protected = set(post_data.keys()).intersection(set(Metadata.get_protected())) + for field in protected: + if getattr(metadata_session, field) != post_data[field]: + self.logger.warning( + "User trying to update the non-updatable fields. {}: {}", + field, + post_data[field], + ) + abort(403, message="Invalid update request payload") + try: + metadata_session.update(**post_data) + self.logger.info( + "User metadata session updated, id: {}", metadata_session.id + ) + except Exception: + self.logger.exception("Exception occurred updating the Metadata model") + abort(500, message="INTERNAL ERROR") + else: + response_object = { + "message": "success", + "data": { + "id": metadata_session.id, + "config": metadata_session.config, + "description": metadata_session.description, + }, + } + return make_response(jsonify(response_object), 200) + + @Auth.token_auth.login_required() + def delete(self, id=None): + """ + Delete request for deleting a metadata session of a user given a metadata id. + This requires a Pbench auth token in the header field + + The url requires a metadata session id such as /user/metadata/ + + Required headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "success" + } + """ + if not id: + self.logger.warning("Metadata id not provided during metadata query") + abort(400, message="Please provide a metadata id to query") + + try: + # Fetch the metadata session + metadata_session = Metadata.query(id=id) + except Exception: + self.logger.exception( + "Exception occurred in the Delete request while querying the Metadata model, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + # Verify if the metadata session id in the url belongs to the logged in user + self.verify_metadata(metadata_session, "delete") + + try: + # Delete the metadata session + Metadata.delete(id=id) + except Exception: + self.logger.exception( + "Exception occurred in the while deleting the metadata entry, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "success", + } + return make_response(jsonify(response_object), 200) diff --git a/lib/pbench/server/database/models/metadata.py b/lib/pbench/server/database/models/metadata.py index 69385d980d..c835fd7945 100644 --- a/lib/pbench/server/database/models/metadata.py +++ b/lib/pbench/server/database/models/metadata.py @@ -1,31 +1,85 @@ import datetime -from dateutil import parser from pbench.server.database.database import Database -from sqlalchemy import Column, Integer, String, DateTime, ForeignKey +from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Text -class MetadataModel(Database.Base): +class Metadata(Database.Base): """ Metadata Model for storing user metadata details """ # TODO: Think about the better name __tablename__ = "metadata" id = Column(Integer, primary_key=True, autoincrement=True) - created = Column(DateTime, nullable=False) - updated = Column(DateTime, nullable=False) - config = Column(String(255), unique=False, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now()) + updated = Column(DateTime, nullable=False, default=datetime.datetime.now()) + config = Column(Text, unique=False, nullable=False) description = Column(String(255), nullable=False) - user_id = Column(Integer, ForeignKey('users.id')) - - def __init__(self, created, config, description, user_id): - self.created = parser.parse(created) - self.updated = datetime.datetime.now() - self.config = config - self.description = description - self.user_id = user_id + user_id = Column(Integer, ForeignKey("users.id")) def __str__(self): return f"Url id: {self.id}, created on: {self.created}, description: {self.description}" - def as_dict(self): - return {c.name: str(getattr(self, c.name)) for c in self.__table__.columns} \ No newline at end of file + def get_json(self): + return { + "id": self.id, + "config": self.config, + "description": self.description, + "created": self.created, + "updated": self.updated, + } + + @staticmethod + def get_protected(): + return ["id", "created"] + + @staticmethod + def query(id=None, user_id=None): + # Currently we would only query with single argument. Argument need to be either id/user_id + if id: + metadata = Database.db_session.query(Metadata).filter_by(id=id).first() + elif user_id: + # If the query parameter is user_id then we return the list of all the metadata linked to that user + metadata = Database.db_session.query(Metadata).filter_by(user_id=user_id) + else: + metadata = None + + return metadata + + def add(self): + """ + Add the current metadata object to the database + """ + try: + Database.db_session.add(self) + Database.db_session.commit() + except Exception: + Database.db_session.rollback() + raise + + def update(self, **kwargs): + """ + Update the current user object with given keyword arguments + """ + try: + for key, value in kwargs.items(): + setattr(self, key, value) + Database.db_session.commit() + except Exception: + Database.db_session.rollback() + raise + + @staticmethod + def delete(id): + """ + Delete the metadata session with a given id + :param username: + :return: + """ + try: + metadata_query = Database.db_session.query(Metadata).filter_by(id=id) + metadata_query.delete() + Database.db_session.commit() + return True + except Exception: + Database.db_session.rollback() + raise diff --git a/lib/pbench/test/unit/server/test_metadata_sessions.py b/lib/pbench/test/unit/server/test_metadata_sessions.py new file mode 100644 index 0000000000..d408086fd7 --- /dev/null +++ b/lib/pbench/test/unit/server/test_metadata_sessions.py @@ -0,0 +1,223 @@ +import datetime +from lib.pbench.test.unit.server.test_user_auth import register_user, login_user + + +def user_register_login(client, server_config): + with client: + response = register_user( + client, + server_config, + username="user", + firstname="firstname", + lastname="lastName", + email="user@domain.com", + password="12345", + ) + data = response.json + assert data["message"] == "Successfully registered." + + response = login_user(client, server_config, "user", "12345") + data_login = response.json + assert data_login["auth_token"] + return data_login + + +class TestMetadataSession: + @staticmethod + def test_metadata_creation(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config2", "description": "description2"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + + response = client.get( + f"{server_config.rest_uri}/metadata", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["sessions"] + assert response.status_code == 200 + + @staticmethod + def test_unauthorized_metadata_creation(client, server_config): + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + ) + data = response.json + assert data is None + assert response.status_code == 401 + + @staticmethod + def test_single_metadata_query(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.get( + f"{server_config.rest_uri}/metadata/{metadata_id}", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["config"] == "config1" + + @staticmethod + def test_unauthorized_metadata_query1(client, server_config): + # Test querying metadata without Pbench auth header + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.get(f"{server_config.rest_uri}/metadata/{metadata_id}",) + assert response.status_code == 401 + + @staticmethod + def test_unauthorized_metadata_query2(client, server_config): + # Test querying someone else's metadata + data_login_1 = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login_1["auth_token"]), + ) + data_1 = response.json + assert data_1["message"] == "success" + assert data_1["data"]["id"] + + # Create another user and login + response = register_user( + client, + server_config, + username="user2", + firstname="firstname2", + lastname="lastName2", + email="user2@domain.com", + password="12345", + ) + data = response.json + assert data["message"] == "Successfully registered." + + response = login_user(client, server_config, "user2", "12345") + data_login_2 = response.json + assert data_login_2["auth_token"] + + # Create metadata session for 2nd user + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config2", "description": "description2"}, + headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), + ) + data_2 = response.json + assert data_2["message"] == "success" + assert data_2["data"]["id"] + + # Query the metadata session id of the 1st user with an auth token of 2nd user + metadata_id = data_1["data"]["id"] + response = client.get( + f"{server_config.rest_uri}/metadata/{metadata_id}", + headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), + ) + data = response.json + assert data["message"] == "Not authorized to perform the specified action" + assert response.status_code == 403 + + @staticmethod + def test_metadata_update(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.put( + f"{server_config.rest_uri}/metadata/{metadata_id}", + json={"description": "description2"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["description"] == "description2" + + @staticmethod + def test_metadata_update_with_invalid_fields(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.put( + f"{server_config.rest_uri}/metadata/{metadata_id}", + json={"created": datetime.datetime.now()}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "Invalid update request payload" + assert response.status_code == 403 + + @staticmethod + def test_metadata_delete(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={"config": "config1", "description": "description1"}, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.delete( + f"{server_config.rest_uri}/metadata/{metadata_id}", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "success" + assert response.status_code == 200 diff --git a/lib/pbench/test/unit/server/test_user_auth.py b/lib/pbench/test/unit/server/test_user_auth.py index 4e2ab6f7b1..7f6923c0b0 100644 --- a/lib/pbench/test/unit/server/test_user_auth.py +++ b/lib/pbench/test/unit/server/test_user_auth.py @@ -423,50 +423,3 @@ def test_delete_user(client, server_config): data = response.json assert data["message"] == "Successfully deleted." assert response.status_code == 200 - - -class TestMetadataSession: - @staticmethod - def test_metadata_creation(client, server_config): - """ Test for user registration """ - with client: - response = register_user( - client, - server_config, - username="user", - firstname="firstname", - lastname="lastName", - email="user@domain.com", - password="12345", - ) - data = response.json - assert data["status"] == "success" - - response = login_user(client, server_config, "user", "12345") - data_login = response.json - assert data_login["status"] == "success" - assert data_login["auth_token"] - - response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, - headers=dict(Authorization="Bearer " + data_login["auth_token"]) - ) - data = response.json - assert data["status"] == "success" - - response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config2", "description": "description2"}, - headers=dict(Authorization="Bearer " + data_login["auth_token"]) - ) - data = response.json - assert data["status"] == "success" - - response = client.get( - f"{server_config.rest_uri}/metadata", - headers=dict(Authorization="Bearer " + data_login["auth_token"]) - ) - data = response.json - assert data["status"] == "success" - assert response.status_code == 200 \ No newline at end of file From cc49270ef8789c08781a0848e8566b063836dcbe Mon Sep 17 00:00:00 2001 From: npalaska Date: Thu, 11 Mar 2021 00:23:02 -0500 Subject: [PATCH 07/11] Add a key and client defined value column to metadata table --- lib/pbench/server/api/__init__.py | 4 +- .../server/api/resources/metadata_api.py | 146 ++++++++++-------- lib/pbench/server/database/models/metadata.py | 43 +++--- .../unit/server/test_metadata_sessions.py | 102 ++++++++---- 4 files changed, 184 insertions(+), 111 deletions(-) diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 5e5d3603db..9647e324ec 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -87,12 +87,12 @@ def register_endpoints(api, app, config): api.add_resource( UserMetadata, - f"{base_uri}/metadata", + f"{base_uri}/metadata/", resource_class_args=(config, logger, token_auth), ) api.add_resource( QueryMetadata, - f"{base_uri}/metadata/", + f"{base_uri}/metadata/", resource_class_args=(config, logger), ) diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py index 8305cdeeb5..71a5f2fb5e 100644 --- a/lib/pbench/server/api/resources/metadata_api.py +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -1,6 +1,6 @@ from flask_restful import Resource, abort from flask import request, make_response, jsonify -from pbench.server.database.models.metadata import Metadata +from pbench.server.database.models.metadata import Metadata, MetadataKeys from pbench.server.api.auth import Auth @@ -14,58 +14,70 @@ def __init__(self, config, logger, auth): self.logger = logger self.auth = auth - @Auth.token_auth.login_required() - def post(self): + @Auth.token_auth.login_required(optional=True) + def post(self, key): """ Post request for creating metadata instance for a user. - This requires a Pbench auth token in the header field + + The url requires a metadata session key such as /metadata/ + the only keys acceptable to create the metadata sessions are favourite/saved This requires a JSON data with required user metadata fields { - "config": "config", - "description": "description", + "value": "blog text" , } + Example: {"value": '{"config": "config", "description": "description"}'} - Required headers include + Authorization header can be included as Authorization: Bearer + If the authorization header is not present, the created metadata session becomes public by default :return: JSON Payload response_object = { "message": "success" "data" { "id": "metadata_id", - "config": "Config string" - "description": "Description string" + "value": "client text blob", + "created": "datetime string", + "updated": "datetime string", + "key": favorite/saved } } """ + metadata_key = key.upper() + if metadata_key not in [key.name for key in MetadataKeys]: + self.logger.warning( + "Invalid Metadata key provided during metadata creation. Key: {}", + metadata_key, + ) + abort( + 400, + message="Invalid metadata key. Key need to be either Favorite/Saved", + ) + post_data = request.get_json() if not post_data: self.logger.warning("Invalid json object: {}", request.url) abort(400, message="Invalid json object in request") - current_user_id = self.auth.token_auth.current_user().id - - config = post_data.get("config") - if not config: - self.logger.warning( - "Config not provided during metadata creation. user_id: {}", - current_user_id, - ) - abort(400, message="Config field missing") + current_user = self.auth.token_auth.current_user() + if current_user: + current_user_id = current_user.id + else: + current_user_id = None - description = post_data.get("description") - if not description: + value = post_data.get("value") + if not value: self.logger.warning( - "Description not provided during metadata creation by user: {}", + "value not provided during metadata creation. user_id: {}", current_user_id, ) - abort(400, message="Description field missing") + abort(400, message="Value field missing") try: # Create a new metadata session metadata_session = Metadata( - config=config, description=description, user_id=current_user_id + value=value, key=metadata_key, user_id=current_user_id ) # insert the metadata session for a user metadata_session.add() @@ -78,16 +90,12 @@ def post(self): else: response_object = { "message": "success", - "data": { - "id": metadata_session.id, - "config": metadata_session.config, - "description": metadata_session.description, - }, + "data": metadata_session.get_json(), } return make_response(jsonify(response_object), 201) - @Auth.token_auth.login_required() - def get(self): + @Auth.token_auth.login_required(optional=True) + def get(self, key): """ Get request for querying all the metadata sessions for a user. returns the list of all the metadata sessions associated with a logged in user. @@ -98,20 +106,38 @@ def get(self): :return: JSON Payload response_object = { - "status": "success" + "message": "success" "data": { "sessions": [ {"id": "metadata_id", - "config": "Config string" - "description": "Description string"}, + "value": "client text blob", + "key": "key string", + "created": "datetime string", + "updated": "datetime string", }, {}] } } """ - current_user_id = self.auth.token_auth.current_user().id + if not key: + self.logger.warning("Metadata key not provided during metadata query") + abort(400, message="Missing metadata key in the query url") + + current_user = self.auth.token_auth.current_user() + if current_user: + current_user_id = current_user.id + else: + current_user_id = None + print(current_user_id) try: - # Fetch the metadata session - metadata_sessions = Metadata.query(user_id=current_user_id) + # Fetch the metadata sessions + # If the key is favorite, we return only favorite sessions, + # else we return all the saved and favorite sessions + if key.upper() == "FAVORITE": + metadata_sessions = Metadata.query( + user_id=current_user_id, key=MetadataKeys[key.upper()].value + ) + else: + metadata_sessions = Metadata.query(user_id=current_user_id) data = [session.get_json() for session in metadata_sessions] except Exception: self.logger.exception( @@ -153,18 +179,20 @@ def get(self, id=None): Get request for querying a metadata session of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata session id such as /user/metadata/ Required headers include Authorization: Bearer :return: JSON Payload response_object = { - "message": "success" - "data" { + "message": "success", + "data": { "id": "metadata_id", - "config": "Config string" - "description": "Description string" + "value": "client text blob" + "created": "Datetime string" + "updated": "Datetime String" + "key": "key string" } } """ @@ -174,7 +202,7 @@ def get(self, id=None): try: # Fetch the metadata session - metadata_session = Metadata.query(id=id) + metadata_session = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the GET request while querying the Metadata model, id: {}", @@ -187,11 +215,7 @@ def get(self, id=None): response_object = { "message": "success", - "data": { - "id": metadata_session.id, - "config": metadata_session.config, - "description": metadata_session.description, - }, + "data": metadata_session.get_json(), } return make_response(jsonify(response_object), 200) @@ -201,23 +225,27 @@ def put(self, id=None): Put request for updating a metadata session of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata session id such as /user/metadata/ This requires a JSON data with required user metadata fields to update { - "description": "description", + "value": "new text value", + ... } + To update the value field, it is required to send the complete text blob again Required headers include Authorization: Bearer :return: JSON Payload response_object = { - "message": "success" - "data" { + "message": "success", + "data": { "id": "metadata_id", - "config": "Config string" - "description": "Description string" + "value": "client text blob" + "created": "Datetime string" + "updated": "Datetime String" + "key": "key string" } } """ @@ -231,7 +259,7 @@ def put(self, id=None): abort(400, message="Invalid json object in request") try: - metadata_session = Metadata.query(id=id) + metadata_session = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the PUT request while querying the Metadata model, id: {}", @@ -274,11 +302,7 @@ def put(self, id=None): else: response_object = { "message": "success", - "data": { - "id": metadata_session.id, - "config": metadata_session.config, - "description": metadata_session.description, - }, + "data": metadata_session.get_json(), } return make_response(jsonify(response_object), 200) @@ -288,7 +312,7 @@ def delete(self, id=None): Delete request for deleting a metadata session of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata session id such as /user/metadata/ Required headers include Authorization: Bearer @@ -304,7 +328,7 @@ def delete(self, id=None): try: # Fetch the metadata session - metadata_session = Metadata.query(id=id) + metadata_session = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the Delete request while querying the Metadata model, id: {}", diff --git a/lib/pbench/server/database/models/metadata.py b/lib/pbench/server/database/models/metadata.py index c835fd7945..cd25d8f149 100644 --- a/lib/pbench/server/database/models/metadata.py +++ b/lib/pbench/server/database/models/metadata.py @@ -1,6 +1,13 @@ +import enum import datetime from pbench.server.database.database import Database -from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Text +from sqlalchemy import Column, Integer, DateTime, ForeignKey, Text +from sqlalchemy.orm import validates + + +class MetadataKeys(enum.Enum): + FAVORITE = 1 + SAVED = 2 class Metadata(Database.Base): @@ -12,38 +19,34 @@ class Metadata(Database.Base): id = Column(Integer, primary_key=True, autoincrement=True) created = Column(DateTime, nullable=False, default=datetime.datetime.now()) updated = Column(DateTime, nullable=False, default=datetime.datetime.now()) - config = Column(Text, unique=False, nullable=False) - description = Column(String(255), nullable=False) - user_id = Column(Integer, ForeignKey("users.id")) + value = Column(Text, unique=False, nullable=False) + key = Column(Integer, nullable=False) + user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=True) - def __str__(self): - return f"Url id: {self.id}, created on: {self.created}, description: {self.description}" + @validates("key") + def evaluate_key(self, key, value): + return MetadataKeys[value].value def get_json(self): return { "id": self.id, - "config": self.config, - "description": self.description, + "value": self.value, "created": self.created, "updated": self.updated, + "key": MetadataKeys(self.key).name, } @staticmethod def get_protected(): - return ["id", "created"] + return ["id", "created", "user_id"] @staticmethod - def query(id=None, user_id=None): - # Currently we would only query with single argument. Argument need to be either id/user_id - if id: - metadata = Database.db_session.query(Metadata).filter_by(id=id).first() - elif user_id: - # If the query parameter is user_id then we return the list of all the metadata linked to that user - metadata = Database.db_session.query(Metadata).filter_by(user_id=user_id) - else: - metadata = None - - return metadata + def query(**kwargs): + query = Database.db_session.query(Metadata) + for attr, value in kwargs.items(): + print(getattr(Metadata, attr), value) + query = query.filter(getattr(Metadata, attr) == value) + return query.all() def add(self): """ diff --git a/lib/pbench/test/unit/server/test_metadata_sessions.py b/lib/pbench/test/unit/server/test_metadata_sessions.py index d408086fd7..5d961c89b1 100644 --- a/lib/pbench/test/unit/server/test_metadata_sessions.py +++ b/lib/pbench/test/unit/server/test_metadata_sessions.py @@ -24,55 +24,98 @@ def user_register_login(client, server_config): class TestMetadataSession: @staticmethod - def test_metadata_creation(client, server_config): + def test_metadata_creation_with_authorization(client, server_config): data_login = user_register_login(client, server_config) with client: + # create a favorite session response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description1"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json assert data["message"] == "success" + # create a saved session response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config2", "description": "description2"}, + f"{server_config.rest_uri}/metadata/saved", + json={"value": '{"config": "config2", "description": "description2"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json assert data["message"] == "success" + # Get all the favorite sessions of logged in user response = client.get( - f"{server_config.rest_uri}/metadata", + f"{server_config.rest_uri}/metadata/favorite", headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) + assert response.status_code == 200 data = response.json assert data["message"] == "success" - assert data["data"]["sessions"] + assert ( + data["data"]["sessions"][0]["value"] + == '{"config": "config1", "description": "description1"}' + ) + + # Get all the saved sessions of logged in user + response = client.get( + f"{server_config.rest_uri}/metadata/saved", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) assert response.status_code == 200 + data = response.json + assert data["message"] == "success" + assert len(data["data"]["sessions"]) == 2 @staticmethod def test_unauthorized_metadata_creation(client, server_config): with client: + # Create a saved session response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/saved", + json={"value": '{"config": "config1", "description": "description1"}'}, ) data = response.json - assert data is None - assert response.status_code == 401 + assert data + assert response.status_code == 201 + + # Create a favorite session + response = client.post( + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config2", "description": "description2"}'}, + ) + data = response.json + assert data["message"] == "success" + + # Get all the favorite sessions of non-logged in user + response = client.get(f"{server_config.rest_uri}/metadata/favorite") + assert response.status_code == 200 + data = response.json + assert data["message"] == "success" + assert ( + data["data"]["sessions"][0]["value"] + == '{"config": "config2", "description": "description2"}' + ) + + # Get all the saved sessions of non-logged in user + response = client.get(f"{server_config.rest_uri}/metadata/saved",) + assert response.status_code == 200 + data = response.json + assert data["message"] == "success" + assert len(data["data"]["sessions"]) == 2 @staticmethod def test_single_metadata_query(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description1"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json + assert response.status_code == 201 assert data["message"] == "success" assert data["data"]["id"] @@ -83,7 +126,7 @@ def test_single_metadata_query(client, server_config): ) data = response.json assert data["message"] == "success" - assert data["data"]["config"] == "config1" + assert data["data"]["key"] == "FAVORITE" @staticmethod def test_unauthorized_metadata_query1(client, server_config): @@ -91,8 +134,8 @@ def test_unauthorized_metadata_query1(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description1"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json @@ -109,8 +152,8 @@ def test_unauthorized_metadata_query2(client, server_config): data_login_1 = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description1"}'}, headers=dict(Authorization="Bearer " + data_login_1["auth_token"]), ) data_1 = response.json @@ -136,8 +179,8 @@ def test_unauthorized_metadata_query2(client, server_config): # Create metadata session for 2nd user response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config2", "description": "description2"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config2", "description": "description2"}'}, headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), ) data_2 = response.json @@ -159,8 +202,8 @@ def test_metadata_update(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description1"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json @@ -170,20 +213,23 @@ def test_metadata_update(client, server_config): metadata_id = data["data"]["id"] response = client.put( f"{server_config.rest_uri}/metadata/{metadata_id}", - json={"description": "description2"}, + json={"value": '{"config": "config1", "description": "description2"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json assert data["message"] == "success" - assert data["data"]["description"] == "description2" + assert ( + data["data"]["value"] + == '{"config": "config1", "description": "description2"}' + ) @staticmethod def test_metadata_update_with_invalid_fields(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description2"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json @@ -205,8 +251,8 @@ def test_metadata_delete(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata", - json={"config": "config1", "description": "description1"}, + f"{server_config.rest_uri}/metadata/favorite", + json={"value": '{"config": "config1", "description": "description2"}'}, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json From c164f18a5252525639dac42419a3a2ca2733ef30 Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 17 Mar 2021 00:39:47 -0400 Subject: [PATCH 08/11] Refactoring and Update metadata API definitions --- lib/pbench/server/api/__init__.py | 13 +- .../server/api/resources/graphql_schema.py | 74 ------- .../server/api/resources/metadata_api.py | 202 ++++++++---------- lib/pbench/server/database/models/metadata.py | 35 +-- .../unit/server/test_metadata_sessions.py | 130 ++++++----- 5 files changed, 192 insertions(+), 262 deletions(-) delete mode 100644 lib/pbench/server/api/resources/graphql_schema.py diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 9647e324ec..940512e78b 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -15,7 +15,11 @@ from pbench.common.exceptions import BadConfig, ConfigFileNotSpecified from pbench.server.api.resources.upload_api import Upload, HostInfo from pbench.server.api.resources.graphql_api import GraphQL -from pbench.server.api.resources.metadata_api import UserMetadata, QueryMetadata +from pbench.server.api.resources.metadata_api import ( + CreateMetadata, + GetMetadata, + QueryMetadata, +) from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch from pbench.server.api.resources.query_apis.query_controllers import QueryControllers @@ -86,7 +90,12 @@ def register_endpoints(api, app, config): ) api.add_resource( - UserMetadata, + CreateMetadata, + f"{base_uri}/metadata", + resource_class_args=(config, logger, token_auth), + ) + api.add_resource( + GetMetadata, f"{base_uri}/metadata/", resource_class_args=(config, logger, token_auth), ) diff --git a/lib/pbench/server/api/resources/graphql_schema.py b/lib/pbench/server/api/resources/graphql_schema.py deleted file mode 100644 index f8fbe0c6eb..0000000000 --- a/lib/pbench/server/api/resources/graphql_schema.py +++ /dev/null @@ -1,74 +0,0 @@ -from pbench.server.api.resources.models import MetadataModel -from pbench.server.api.resources.database import Database - -import graphene -from graphene import relay -from graphene_sqlalchemy import SQLAlchemyObjectType - - -# Define graphql types -class Metadata(SQLAlchemyObjectType): - class Meta: - model = MetadataModel - interfaces = (relay.Node,) - - -class MetadataAttribute: - id = graphene.ID - user_id = graphene.ID() - created = graphene.DateTime() - updated = graphene.DateTime() - config = graphene.String() - description = graphene.String() - - -class CreateMetadataInput(graphene.InputObjectType, MetadataAttribute): - pass - - -# mutations -class CreateMetadata(graphene.Mutation): - metadata = graphene.Field(lambda: Metadata) - ok = graphene.Boolean() - - class Arguments: - input = CreateMetadataInput(required=True) - - @staticmethod - def mutate(self, info, input): - data = input - metadata = MetadataModel(**data) - Database.db_session.add(metadata) - Database.db_session.commit() - ok = True - return CreateMetadata(metadata=metadata, ok=ok) - - -class Mutation(graphene.ObjectType): - createMetadata = CreateMetadata.Field() - - -# Query -class Query(graphene.ObjectType): - node = relay.Node.Field() - - metadata_by_id = graphene.List(Metadata, id=graphene.String()) - metadata_by_userid = graphene.List(Metadata, userid=graphene.String()) - - @staticmethod - def resolve_metadata_by_id(parent, info, **args): - q = args.get("id") - - metadata_query = Metadata.get_query(info) - return metadata_query.filter(MetadataModel.id == q).all() - - @staticmethod - def resolve_metadata_by_userid(parent, info, **args): - q = args.get("userid") - - metadata_query = Metadata.get_query(info) - return metadata_query.filter(MetadataModel.user_id == q).all() - - -# schema -schema = graphene.Schema(query=Query, mutation=Mutation, types=[Metadata]) diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py index 71a5f2fb5e..3b6f5ae496 100644 --- a/lib/pbench/server/api/resources/metadata_api.py +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -1,12 +1,12 @@ from flask_restful import Resource, abort from flask import request, make_response, jsonify -from pbench.server.database.models.metadata import Metadata, MetadataKeys +from pbench.server.database.models.metadata import Metadata from pbench.server.api.auth import Auth -class UserMetadata(Resource): +class CreateMetadata(Resource): """ - Abstracted pbench API for handling user metadata by using graphql schema + Abstracted pbench API for handling user metadata """ def __init__(self, config, logger, auth): @@ -15,130 +15,121 @@ def __init__(self, config, logger, auth): self.auth = auth @Auth.token_auth.login_required(optional=True) - def post(self, key): + def post(self): """ Post request for creating metadata instance for a user. - The url requires a metadata session key such as /metadata/ - the only keys acceptable to create the metadata sessions are favourite/saved + The url requires a metadata object key such as /metadata/ This requires a JSON data with required user metadata fields { + "key": "Metadata_key" # e.g favorite/saved_view "value": "blog text" , } - Example: {"value": '{"config": "config", "description": "description"}'} Authorization header can be included as Authorization: Bearer - If the authorization header is not present, the created metadata session becomes public by default + If the authorization header is not present, the created metadata object becomes public by default :return: JSON Payload response_object = { - "message": "success" "data" { "id": "metadata_id", - "value": "client text blob", "created": "datetime string", "updated": "datetime string", - "key": favorite/saved + "key": "user defined key" } } """ - metadata_key = key.upper() - if metadata_key not in [key.name for key in MetadataKeys]: - self.logger.warning( - "Invalid Metadata key provided during metadata creation. Key: {}", - metadata_key, - ) - abort( - 400, - message="Invalid metadata key. Key need to be either Favorite/Saved", - ) - post_data = request.get_json() if not post_data: self.logger.warning("Invalid json object: {}", request.url) abort(400, message="Invalid json object in request") + value = post_data.get("value") + if value is None: + self.logger.warning("Value not provided during metadata creation.") + abort(400, message="Value field missing") + + metadata_key = post_data.get("key") + if metadata_key is None: + self.logger.warning("Key not provided during metadata creation.") + abort(400, message="Key field missing") + current_user = self.auth.token_auth.current_user() if current_user: current_user_id = current_user.id else: current_user_id = None - value = post_data.get("value") - if not value: - self.logger.warning( - "value not provided during metadata creation. user_id: {}", - current_user_id, - ) - abort(400, message="Value field missing") - try: - # Create a new metadata session - metadata_session = Metadata( - value=value, key=metadata_key, user_id=current_user_id + # Create a new metadata object + metadata_object = Metadata( + value=value, key=metadata_key.lower(), user_id=current_user_id ) - # insert the metadata session for a user - metadata_session.add() + # insert the metadata object for a user + metadata_object.add() self.logger.info( - "New metadata session created for user_id {}", current_user_id + "New metadata object created for user_id {}", current_user_id ) except Exception: self.logger.exception("Exception occurred during the Metadata creation") abort(500, message="INTERNAL ERROR") else: response_object = { - "message": "success", - "data": metadata_session.get_json(), + "data": metadata_object.get_json( + include=["id", "created", "updated", "key"] + ), } return make_response(jsonify(response_object), 201) + +class GetMetadata(Resource): + """ + Abstracted pbench API for handling user metadata + """ + + def __init__(self, config, logger, auth): + self.server_config = config + self.logger = logger + self.auth = auth + @Auth.token_auth.login_required(optional=True) def get(self, key): """ - Get request for querying all the metadata sessions for a user. - returns the list of all the metadata sessions associated with a logged in user. + Get request for querying all the metadata objects for a user. + returns the list of all the metadata objects of a specified key associated with a logged in user. + If the user is not logged in we return all the public metadata objects of a specified key. This requires a Pbench auth token in the header field - Required headers include + Optional headers include Authorization: Bearer :return: JSON Payload response_object = { - "message": "success" - "data": { - "sessions": [ + "data": [ {"id": "metadata_id", "value": "client text blob", - "key": "key string", "created": "datetime string", - "updated": "datetime string", }, - {}] - } + "updated": "datetime string", }, ...] } """ - if not key: + if key is None: self.logger.warning("Metadata key not provided during metadata query") - abort(400, message="Missing metadata key in the query url") + abort(400, message="Missing metadata key in the get request uri") current_user = self.auth.token_auth.current_user() if current_user: current_user_id = current_user.id else: current_user_id = None - print(current_user_id) try: - # Fetch the metadata sessions - # If the key is favorite, we return only favorite sessions, - # else we return all the saved and favorite sessions - if key.upper() == "FAVORITE": - metadata_sessions = Metadata.query( - user_id=current_user_id, key=MetadataKeys[key.upper()].value - ) - else: - metadata_sessions = Metadata.query(user_id=current_user_id) - data = [session.get_json() for session in metadata_sessions] + # Query the metadata object with a given key + metadata_objects = Metadata.query(user_id=current_user_id, key=key.lower()) + data = [ + metadata.get_json(include=["id", "created", "updated", "value"]) + for metadata in metadata_objects + ] except Exception: self.logger.exception( "Exception occurred while querying the Metadata model" @@ -146,47 +137,45 @@ def get(self, key): abort(500, message="INTERNAL ERROR") response_object = { - "message": "success", - "data": {"sessions": data}, + "data": data, } return make_response(jsonify(response_object), 200) class QueryMetadata(Resource): """ - Abstracted pbench API for querying a single user metadata session + Abstracted pbench API for querying a single user metadata object """ def __init__(self, config, logger): self.server_config = config self.logger = logger - def verify_metadata(self, session, action): - current_user_id = Auth.token_auth.current_user().id - metadata_user_id = session.user_id - if current_user_id != metadata_user_id: + def verify_metadata(self, metadata): + current_user = Auth.token_auth.current_user() + metadata_user_id = metadata.user_id + if current_user.id != metadata_user_id and not current_user.is_admin(): self.logger.warning( - "Metadata {}: Logged in user_id {} is different than the one provided in the URI {}", - action, - current_user_id, + "Metadata user verification: Logged in user_id {} is different than the one provided in the URI {}", + current_user.id, metadata_user_id, ) - abort(403, message="Not authorized to perform the specified action") + return False + return True @Auth.token_auth.login_required() def get(self, id=None): """ - Get request for querying a metadata session of a user given a metadata id. + Get request for querying a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata object id such as /user/metadata/ Required headers include Authorization: Bearer :return: JSON Payload response_object = { - "message": "success", "data": { "id": "metadata_id", "value": "client text blob" @@ -196,13 +185,13 @@ def get(self, id=None): } } """ - if not id: + if id is None: self.logger.warning("Metadata id not provided during metadata query") - abort(400, message="Please provide a metadata id to query") + abort(400, message="Missing metadata id in the URI") try: - # Fetch the metadata session - metadata_session = Metadata.query(id=id)[0] + # Fetch the metadata object + metadata_object = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the GET request while querying the Metadata model, id: {}", @@ -210,22 +199,24 @@ def get(self, id=None): ) abort(500, message="INTERNAL ERROR") - # Verify if the metadata session id in the url belongs to the logged in user - self.verify_metadata(metadata_session, "get") + # Verify if the metadata object id in the url belongs to the logged in user + if not self.verify_metadata(metadata_object): + abort(403, message="Not authorized to perform the GET request") response_object = { - "message": "success", - "data": metadata_session.get_json(), + "data": metadata_object.get_json( + include=["id", "value", "created", "updated", "key"] + ), } return make_response(jsonify(response_object), 200) @Auth.token_auth.login_required() def put(self, id=None): """ - Put request for updating a metadata session of a user given a metadata id. + Put request for updating a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata object id such as /user/metadata/ This requires a JSON data with required user metadata fields to update { @@ -239,17 +230,15 @@ def put(self, id=None): :return: JSON Payload response_object = { - "message": "success", "data": { "id": "metadata_id", - "value": "client text blob" "created": "Datetime string" "updated": "Datetime String" "key": "key string" } } """ - if not id: + if id is None: self.logger.warning("Metadata id not provided during metadata query") abort(400, message="Please provide a metadata id to query") @@ -259,7 +248,7 @@ def put(self, id=None): abort(400, message="Invalid json object in request") try: - metadata_session = Metadata.query(id=id)[0] + metadata_object = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the PUT request while querying the Metadata model, id: {}", @@ -267,8 +256,9 @@ def put(self, id=None): ) abort(500, message="INTERNAL ERROR") - # Verify if the metadata session id in the url belongs to the logged in user - self.verify_metadata(metadata_session, "put") + # Verify if the metadata object id in the url belongs to the logged in user + if not self.verify_metadata(metadata_object): + abort(403, message="Not authorized to perform the PUT request") # Check if the metadata payload contain fields that are either protected or # not present in the metadata db. If any key in the payload does not match @@ -284,7 +274,7 @@ def put(self, id=None): abort(400, message="Invalid fields in update request payload") protected = set(post_data.keys()).intersection(set(Metadata.get_protected())) for field in protected: - if getattr(metadata_session, field) != post_data[field]: + if getattr(metadata_object, field) != post_data[field]: self.logger.warning( "User trying to update the non-updatable fields. {}: {}", field, @@ -292,27 +282,24 @@ def put(self, id=None): ) abort(403, message="Invalid update request payload") try: - metadata_session.update(**post_data) - self.logger.info( - "User metadata session updated, id: {}", metadata_session.id - ) + metadata_object.update(**post_data) + self.logger.info("User metadata object updated, id: {}", metadata_object.id) except Exception: self.logger.exception("Exception occurred updating the Metadata model") abort(500, message="INTERNAL ERROR") else: response_object = { - "message": "success", - "data": metadata_session.get_json(), + "data": metadata_object.get_json(["id", "created", "updated", "key"]), } return make_response(jsonify(response_object), 200) @Auth.token_auth.login_required() def delete(self, id=None): """ - Delete request for deleting a metadata session of a user given a metadata id. + Delete request for deleting a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field - The url requires a metadata session id such as /user/metadata/ + The url requires a metadata object id such as /user/metadata/ Required headers include Authorization: Bearer @@ -322,13 +309,13 @@ def delete(self, id=None): "message": "success" } """ - if not id: + if id is None: self.logger.warning("Metadata id not provided during metadata query") abort(400, message="Please provide a metadata id to query") try: - # Fetch the metadata session - metadata_session = Metadata.query(id=id)[0] + # Fetch the metadata object + metadata_object = Metadata.query(id=id)[0] except Exception: self.logger.exception( "Exception occurred in the Delete request while querying the Metadata model, id: {}", @@ -336,11 +323,12 @@ def delete(self, id=None): ) abort(500, message="INTERNAL ERROR") - # Verify if the metadata session id in the url belongs to the logged in user - self.verify_metadata(metadata_session, "delete") + # Verify if the metadata object id in the url belongs to the logged in user + if not self.verify_metadata(metadata_object): + abort(403, message="Not authorized to perform the DELETE request") try: - # Delete the metadata session + # Delete the metadata object Metadata.delete(id=id) except Exception: self.logger.exception( @@ -350,6 +338,6 @@ def delete(self, id=None): abort(500, message="INTERNAL ERROR") response_object = { - "message": "success", + "message": "Metadata object deleted", } return make_response(jsonify(response_object), 200) diff --git a/lib/pbench/server/database/models/metadata.py b/lib/pbench/server/database/models/metadata.py index cd25d8f149..1d8bd99088 100644 --- a/lib/pbench/server/database/models/metadata.py +++ b/lib/pbench/server/database/models/metadata.py @@ -1,13 +1,6 @@ -import enum import datetime from pbench.server.database.database import Database -from sqlalchemy import Column, Integer, DateTime, ForeignKey, Text -from sqlalchemy.orm import validates - - -class MetadataKeys(enum.Enum): - FAVORITE = 1 - SAVED = 2 +from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Text class Metadata(Database.Base): @@ -20,21 +13,14 @@ class Metadata(Database.Base): created = Column(DateTime, nullable=False, default=datetime.datetime.now()) updated = Column(DateTime, nullable=False, default=datetime.datetime.now()) value = Column(Text, unique=False, nullable=False) - key = Column(Integer, nullable=False) + key = Column(String(128), nullable=False) user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=True) - @validates("key") - def evaluate_key(self, key, value): - return MetadataKeys[value].value - - def get_json(self): - return { - "id": self.id, - "value": self.value, - "created": self.created, - "updated": self.updated, - "key": MetadataKeys(self.key).name, - } + def get_json(self, include): + data = {} + for key in include: + data.update({key: getattr(self, key)}) + return data @staticmethod def get_protected(): @@ -44,7 +30,6 @@ def get_protected(): def query(**kwargs): query = Database.db_session.query(Metadata) for attr, value in kwargs.items(): - print(getattr(Metadata, attr), value) query = query.filter(getattr(Metadata, attr) == value) return query.all() @@ -61,7 +46,7 @@ def add(self): def update(self, **kwargs): """ - Update the current user object with given keyword arguments + Update the current metadata object with given keyword arguments """ try: for key, value in kwargs.items(): @@ -74,8 +59,8 @@ def update(self, **kwargs): @staticmethod def delete(id): """ - Delete the metadata session with a given id - :param username: + Delete the metadata object with a given id + :param id: metadata_object_id :return: """ try: diff --git a/lib/pbench/test/unit/server/test_metadata_sessions.py b/lib/pbench/test/unit/server/test_metadata_sessions.py index 5d961c89b1..1c94f7d830 100644 --- a/lib/pbench/test/unit/server/test_metadata_sessions.py +++ b/lib/pbench/test/unit/server/test_metadata_sessions.py @@ -27,34 +27,41 @@ class TestMetadataSession: def test_metadata_creation_with_authorization(client, server_config): data_login = user_register_login(client, server_config) with client: - # create a favorite session + # create a favorite object response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description1"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) + assert response.status_code == 201 data = response.json - assert data["message"] == "success" + assert data["data"]["key"] == "favorite" - # create a saved session + # create a saved metadata object response = client.post( - f"{server_config.rest_uri}/metadata/saved", - json={"value": '{"config": "config2", "description": "description2"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "saved", + "value": '{"config": "config2", "description": "description2"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) + assert response.status_code == 201 data = response.json - assert data["message"] == "success" + assert data["data"]["key"] == "saved" - # Get all the favorite sessions of logged in user + # Get all the favorite metadata objects of logged in user response = client.get( f"{server_config.rest_uri}/metadata/favorite", headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) assert response.status_code == 200 data = response.json - assert data["message"] == "success" assert ( - data["data"]["sessions"][0]["value"] + data["data"][0]["value"] == '{"config": "config1", "description": "description1"}' ) @@ -65,16 +72,18 @@ def test_metadata_creation_with_authorization(client, server_config): ) assert response.status_code == 200 data = response.json - assert data["message"] == "success" - assert len(data["data"]["sessions"]) == 2 + assert len(data["data"]) == 1 @staticmethod def test_unauthorized_metadata_creation(client, server_config): with client: # Create a saved session response = client.post( - f"{server_config.rest_uri}/metadata/saved", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "saved", + "value": '{"config": "config1", "description": "description1"}', + }, ) data = response.json assert data @@ -82,19 +91,22 @@ def test_unauthorized_metadata_creation(client, server_config): # Create a favorite session response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config2", "description": "description2"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config2", "description": "description2"}', + }, ) + assert response.status_code == 201 data = response.json - assert data["message"] == "success" + assert data["data"]["key"] == "favorite" # Get all the favorite sessions of non-logged in user response = client.get(f"{server_config.rest_uri}/metadata/favorite") assert response.status_code == 200 data = response.json - assert data["message"] == "success" assert ( - data["data"]["sessions"][0]["value"] + data["data"][0]["value"] == '{"config": "config2", "description": "description2"}' ) @@ -102,21 +114,22 @@ def test_unauthorized_metadata_creation(client, server_config): response = client.get(f"{server_config.rest_uri}/metadata/saved",) assert response.status_code == 200 data = response.json - assert data["message"] == "success" - assert len(data["data"]["sessions"]) == 2 + assert len(data["data"]) == 1 @staticmethod def test_single_metadata_query(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description1"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json assert response.status_code == 201 - assert data["message"] == "success" assert data["data"]["id"] metadata_id = data["data"]["id"] @@ -125,8 +138,7 @@ def test_single_metadata_query(client, server_config): headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json - assert data["message"] == "success" - assert data["data"]["key"] == "FAVORITE" + assert data["data"]["key"] == "favorite" @staticmethod def test_unauthorized_metadata_query1(client, server_config): @@ -134,12 +146,14 @@ def test_unauthorized_metadata_query1(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description1"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json - assert data["message"] == "success" assert data["data"]["id"] metadata_id = data["data"]["id"] @@ -152,12 +166,14 @@ def test_unauthorized_metadata_query2(client, server_config): data_login_1 = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description1"}', + }, headers=dict(Authorization="Bearer " + data_login_1["auth_token"]), ) data_1 = response.json - assert data_1["message"] == "success" assert data_1["data"]["id"] # Create another user and login @@ -179,12 +195,14 @@ def test_unauthorized_metadata_query2(client, server_config): # Create metadata session for 2nd user response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config2", "description": "description2"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config2", "description": "description2"}', + }, headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), ) data_2 = response.json - assert data_2["message"] == "success" assert data_2["data"]["id"] # Query the metadata session id of the 1st user with an auth token of 2nd user @@ -194,7 +212,7 @@ def test_unauthorized_metadata_query2(client, server_config): headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), ) data = response.json - assert data["message"] == "Not authorized to perform the specified action" + assert data["message"] == "Not authorized to perform the GET request" assert response.status_code == 403 @staticmethod @@ -202,13 +220,16 @@ def test_metadata_update(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description1"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description1"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) + assert response.status_code == 201 data = response.json - assert data["message"] == "success" - assert data["data"]["id"] + assert data["data"]["key"] == "favorite" metadata_id = data["data"]["id"] response = client.put( @@ -217,24 +238,23 @@ def test_metadata_update(client, server_config): headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json - assert data["message"] == "success" - assert ( - data["data"]["value"] - == '{"config": "config1", "description": "description2"}' - ) + assert data["data"]["key"] == "favorite" @staticmethod def test_metadata_update_with_invalid_fields(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description2"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description2"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) + assert response.status_code == 201 data = response.json - assert data["message"] == "success" - assert data["data"]["id"] + assert data["data"]["key"] == "favorite" metadata_id = data["data"]["id"] response = client.put( @@ -251,12 +271,14 @@ def test_metadata_delete(client, server_config): data_login = user_register_login(client, server_config) with client: response = client.post( - f"{server_config.rest_uri}/metadata/favorite", - json={"value": '{"config": "config1", "description": "description2"}'}, + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description2"}', + }, headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json - assert data["message"] == "success" assert data["data"]["id"] metadata_id = data["data"]["id"] @@ -265,5 +287,5 @@ def test_metadata_delete(client, server_config): headers=dict(Authorization="Bearer " + data_login["auth_token"]), ) data = response.json - assert data["message"] == "success" + assert data["message"] == "Metadata object deleted" assert response.status_code == 200 From f360de10de584b48f6cbd82c6c8d009c191f1279 Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 17 Mar 2021 01:10:51 -0400 Subject: [PATCH 09/11] make login optional on public metadata obbject query --- .../server/api/resources/metadata_api.py | 41 +++++++++++++++---- ...a_sessions.py => test_metadata_objects.py} | 18 ++++---- 2 files changed, 42 insertions(+), 17 deletions(-) rename lib/pbench/test/unit/server/{test_metadata_sessions.py => test_metadata_objects.py} (95%) diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py index 3b6f5ae496..e4bd565948 100644 --- a/lib/pbench/server/api/resources/metadata_api.py +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -154,6 +154,15 @@ def __init__(self, config, logger): def verify_metadata(self, metadata): current_user = Auth.token_auth.current_user() metadata_user_id = metadata.user_id + if current_user is None: + # The request is not from a logged-in user + if metadata_user_id is None: + return True + self.logger.warning( + "Metadata user verification: Public user is trying to access private metadata object for user {}", + metadata_user_id, + ) + return False if current_user.id != metadata_user_id and not current_user.is_admin(): self.logger.warning( "Metadata user verification: Logged in user_id {} is different than the one provided in the URI {}", @@ -163,15 +172,16 @@ def verify_metadata(self, metadata): return False return True - @Auth.token_auth.login_required() + @Auth.token_auth.login_required(optional=True) def get(self, id=None): """ Get request for querying a metadata object of a user given a metadata id. - This requires a Pbench auth token in the header field + This requires a Pbench auth token in the header field if the metadata object is private to a user + The url requires a metadata object id such as /user/metadata/ - Required headers include + Optional headers include Authorization: Bearer :return: JSON Payload @@ -191,7 +201,7 @@ def get(self, id=None): try: # Fetch the metadata object - metadata_object = Metadata.query(id=id)[0] + metadata_objects = Metadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the GET request while querying the Metadata model, id: {}", @@ -199,6 +209,11 @@ def get(self, id=None): ) abort(500, message="INTERNAL ERROR") + if metadata_objects: + metadata_object = metadata_objects[0] + else: + abort(404, message="Not found") + # Verify if the metadata object id in the url belongs to the logged in user if not self.verify_metadata(metadata_object): abort(403, message="Not authorized to perform the GET request") @@ -210,7 +225,7 @@ def get(self, id=None): } return make_response(jsonify(response_object), 200) - @Auth.token_auth.login_required() + @Auth.token_auth.login_required(optional=True) def put(self, id=None): """ Put request for updating a metadata object of a user given a metadata id. @@ -248,7 +263,7 @@ def put(self, id=None): abort(400, message="Invalid json object in request") try: - metadata_object = Metadata.query(id=id)[0] + metadata_objects = Metadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the PUT request while querying the Metadata model, id: {}", @@ -256,6 +271,11 @@ def put(self, id=None): ) abort(500, message="INTERNAL ERROR") + if metadata_objects: + metadata_object = metadata_objects[0] + else: + abort(404, message="Not found") + # Verify if the metadata object id in the url belongs to the logged in user if not self.verify_metadata(metadata_object): abort(403, message="Not authorized to perform the PUT request") @@ -293,7 +313,7 @@ def put(self, id=None): } return make_response(jsonify(response_object), 200) - @Auth.token_auth.login_required() + @Auth.token_auth.login_required(optional=True) def delete(self, id=None): """ Delete request for deleting a metadata object of a user given a metadata id. @@ -315,7 +335,7 @@ def delete(self, id=None): try: # Fetch the metadata object - metadata_object = Metadata.query(id=id)[0] + metadata_objects = Metadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the Delete request while querying the Metadata model, id: {}", @@ -323,6 +343,11 @@ def delete(self, id=None): ) abort(500, message="INTERNAL ERROR") + if metadata_objects: + metadata_object = metadata_objects[0] + else: + abort(404, message="Not found") + # Verify if the metadata object id in the url belongs to the logged in user if not self.verify_metadata(metadata_object): abort(403, message="Not authorized to perform the DELETE request") diff --git a/lib/pbench/test/unit/server/test_metadata_sessions.py b/lib/pbench/test/unit/server/test_metadata_objects.py similarity index 95% rename from lib/pbench/test/unit/server/test_metadata_sessions.py rename to lib/pbench/test/unit/server/test_metadata_objects.py index 1c94f7d830..a414b944e1 100644 --- a/lib/pbench/test/unit/server/test_metadata_sessions.py +++ b/lib/pbench/test/unit/server/test_metadata_objects.py @@ -22,7 +22,7 @@ def user_register_login(client, server_config): return data_login -class TestMetadataSession: +class TestMetadataObjects: @staticmethod def test_metadata_creation_with_authorization(client, server_config): data_login = user_register_login(client, server_config) @@ -65,7 +65,7 @@ def test_metadata_creation_with_authorization(client, server_config): == '{"config": "config1", "description": "description1"}' ) - # Get all the saved sessions of logged in user + # Get all the saved metadata objects of logged in user response = client.get( f"{server_config.rest_uri}/metadata/saved", headers=dict(Authorization="Bearer " + data_login["auth_token"]), @@ -77,7 +77,7 @@ def test_metadata_creation_with_authorization(client, server_config): @staticmethod def test_unauthorized_metadata_creation(client, server_config): with client: - # Create a saved session + # Create a saved object response = client.post( f"{server_config.rest_uri}/metadata", json={ @@ -89,7 +89,7 @@ def test_unauthorized_metadata_creation(client, server_config): assert data assert response.status_code == 201 - # Create a favorite session + # Create a favorite metadata object response = client.post( f"{server_config.rest_uri}/metadata", json={ @@ -101,7 +101,7 @@ def test_unauthorized_metadata_creation(client, server_config): data = response.json assert data["data"]["key"] == "favorite" - # Get all the favorite sessions of non-logged in user + # Get all the favorite metadata objects of non-logged in user response = client.get(f"{server_config.rest_uri}/metadata/favorite") assert response.status_code == 200 data = response.json @@ -110,7 +110,7 @@ def test_unauthorized_metadata_creation(client, server_config): == '{"config": "config2", "description": "description2"}' ) - # Get all the saved sessions of non-logged in user + # Get all the saved metadata objects of non-logged in user response = client.get(f"{server_config.rest_uri}/metadata/saved",) assert response.status_code == 200 data = response.json @@ -158,7 +158,7 @@ def test_unauthorized_metadata_query1(client, server_config): metadata_id = data["data"]["id"] response = client.get(f"{server_config.rest_uri}/metadata/{metadata_id}",) - assert response.status_code == 401 + assert response.status_code == 403 @staticmethod def test_unauthorized_metadata_query2(client, server_config): @@ -193,7 +193,7 @@ def test_unauthorized_metadata_query2(client, server_config): data_login_2 = response.json assert data_login_2["auth_token"] - # Create metadata session for 2nd user + # Create metadata objects for 2nd user response = client.post( f"{server_config.rest_uri}/metadata", json={ @@ -205,7 +205,7 @@ def test_unauthorized_metadata_query2(client, server_config): data_2 = response.json assert data_2["data"]["id"] - # Query the metadata session id of the 1st user with an auth token of 2nd user + # Query the metadata object id of the 1st user with an auth token of 2nd user metadata_id = data_1["data"]["id"] response = client.get( f"{server_config.rest_uri}/metadata/{metadata_id}", From 9ea52685b5bb6856bbc0c9ddd093f9e6eaa93fcc Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 17 Mar 2021 01:38:48 -0400 Subject: [PATCH 10/11] update user metadata table name --- .../server/api/resources/metadata_api.py | 54 +++++++++++-------- lib/pbench/server/database/models/metadata.py | 10 ++-- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py index e4bd565948..6641d31496 100644 --- a/lib/pbench/server/api/resources/metadata_api.py +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -1,6 +1,6 @@ from flask_restful import Resource, abort from flask import request, make_response, jsonify -from pbench.server.database.models.metadata import Metadata +from pbench.server.database.models.metadata import UserMetadata from pbench.server.api.auth import Auth @@ -46,25 +46,31 @@ def post(self): self.logger.warning("Invalid json object: {}", request.url) abort(400, message="Invalid json object in request") + current_user = self.auth.token_auth.current_user() + if current_user: + current_user_id = current_user.id + else: + current_user_id = None + value = post_data.get("value") if value is None: - self.logger.warning("Value not provided during metadata creation.") + self.logger.warning( + "Value not provided during metadata creation. User_id: {}", + current_user_id, + ) abort(400, message="Value field missing") metadata_key = post_data.get("key") if metadata_key is None: - self.logger.warning("Key not provided during metadata creation.") + self.logger.warning( + "Key not provided during metadata creation. User_id: {}", + current_user_id, + ) abort(400, message="Key field missing") - current_user = self.auth.token_auth.current_user() - if current_user: - current_user_id = current_user.id - else: - current_user_id = None - try: # Create a new metadata object - metadata_object = Metadata( + metadata_object = UserMetadata( value=value, key=metadata_key.lower(), user_id=current_user_id ) # insert the metadata object for a user @@ -125,7 +131,9 @@ def get(self, key): current_user_id = None try: # Query the metadata object with a given key - metadata_objects = Metadata.query(user_id=current_user_id, key=key.lower()) + metadata_objects = UserMetadata.query( + user_id=current_user_id, key=key.lower() + ) data = [ metadata.get_json(include=["id", "created", "updated", "value"]) for metadata in metadata_objects @@ -173,7 +181,7 @@ def verify_metadata(self, metadata): return True @Auth.token_auth.login_required(optional=True) - def get(self, id=None): + def get(self, id): """ Get request for querying a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field if the metadata object is private to a user @@ -201,7 +209,7 @@ def get(self, id=None): try: # Fetch the metadata object - metadata_objects = Metadata.query(id=id) + metadata_objects = UserMetadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the GET request while querying the Metadata model, id: {}", @@ -226,7 +234,7 @@ def get(self, id=None): return make_response(jsonify(response_object), 200) @Auth.token_auth.login_required(optional=True) - def put(self, id=None): + def put(self, id): """ Put request for updating a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field @@ -255,7 +263,7 @@ def put(self, id=None): """ if id is None: self.logger.warning("Metadata id not provided during metadata query") - abort(400, message="Please provide a metadata id to query") + abort(400, message="Missing metadata id in the URI for update operation") post_data = request.get_json() if not post_data: @@ -263,7 +271,7 @@ def put(self, id=None): abort(400, message="Invalid json object in request") try: - metadata_objects = Metadata.query(id=id) + metadata_objects = UserMetadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the PUT request while querying the Metadata model, id: {}", @@ -284,7 +292,7 @@ def put(self, id=None): # not present in the metadata db. If any key in the payload does not match # with the column name we will abort the update request. non_existent = set(post_data.keys()).difference( - set(Metadata.__table__.columns.keys()) + set(UserMetadata.__table__.columns.keys()) ) if non_existent: self.logger.warning( @@ -292,7 +300,9 @@ def put(self, id=None): non_existent, ) abort(400, message="Invalid fields in update request payload") - protected = set(post_data.keys()).intersection(set(Metadata.get_protected())) + protected = set(post_data.keys()).intersection( + set(UserMetadata.get_protected()) + ) for field in protected: if getattr(metadata_object, field) != post_data[field]: self.logger.warning( @@ -314,7 +324,7 @@ def put(self, id=None): return make_response(jsonify(response_object), 200) @Auth.token_auth.login_required(optional=True) - def delete(self, id=None): + def delete(self, id): """ Delete request for deleting a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field @@ -331,11 +341,11 @@ def delete(self, id=None): """ if id is None: self.logger.warning("Metadata id not provided during metadata query") - abort(400, message="Please provide a metadata id to query") + abort(400, message="Missing metadata id in the URI for delete operation") try: # Fetch the metadata object - metadata_objects = Metadata.query(id=id) + metadata_objects = UserMetadata.query(id=id) except Exception: self.logger.exception( "Exception occurred in the Delete request while querying the Metadata model, id: {}", @@ -354,7 +364,7 @@ def delete(self, id=None): try: # Delete the metadata object - Metadata.delete(id=id) + UserMetadata.delete(id=id) except Exception: self.logger.exception( "Exception occurred in the while deleting the metadata entry, id: {}", diff --git a/lib/pbench/server/database/models/metadata.py b/lib/pbench/server/database/models/metadata.py index 1d8bd99088..ee5ea014ff 100644 --- a/lib/pbench/server/database/models/metadata.py +++ b/lib/pbench/server/database/models/metadata.py @@ -3,11 +3,11 @@ from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Text -class Metadata(Database.Base): +class UserMetadata(Database.Base): """ Metadata Model for storing user metadata details """ # TODO: Think about the better name - __tablename__ = "metadata" + __tablename__ = "user_metadata" id = Column(Integer, primary_key=True, autoincrement=True) created = Column(DateTime, nullable=False, default=datetime.datetime.now()) @@ -28,9 +28,9 @@ def get_protected(): @staticmethod def query(**kwargs): - query = Database.db_session.query(Metadata) + query = Database.db_session.query(UserMetadata) for attr, value in kwargs.items(): - query = query.filter(getattr(Metadata, attr) == value) + query = query.filter(getattr(UserMetadata, attr) == value) return query.all() def add(self): @@ -64,7 +64,7 @@ def delete(id): :return: """ try: - metadata_query = Database.db_session.query(Metadata).filter_by(id=id) + metadata_query = Database.db_session.query(UserMetadata).filter_by(id=id) metadata_query.delete() Database.db_session.commit() return True From 0f0c58a25707063f8728d406eb3e4ae70f5dcefd Mon Sep 17 00:00:00 2001 From: npalaska Date: Wed, 17 Mar 2021 19:32:40 -0400 Subject: [PATCH 11/11] Add strict auth req on createMetadata, add api for publishing metadata --- lib/pbench/server/api/__init__.py | 6 + .../server/api/resources/metadata_api.py | 151 +++++++++++++----- .../test/unit/server/test_metadata_objects.py | 51 ++++-- 3 files changed, 152 insertions(+), 56 deletions(-) diff --git a/lib/pbench/server/api/__init__.py b/lib/pbench/server/api/__init__.py index 940512e78b..3a54c6a1b7 100644 --- a/lib/pbench/server/api/__init__.py +++ b/lib/pbench/server/api/__init__.py @@ -19,6 +19,7 @@ CreateMetadata, GetMetadata, QueryMetadata, + PublishMetadata, ) from pbench.common.logger import get_pbench_logger from pbench.server.api.resources.query_apis.elasticsearch_api import Elasticsearch @@ -104,6 +105,11 @@ def register_endpoints(api, app, config): f"{base_uri}/metadata/", resource_class_args=(config, logger), ) + api.add_resource( + PublishMetadata, + f"{base_uri}/metadata//publish", + resource_class_args=(config, logger), + ) def get_server_config(): diff --git a/lib/pbench/server/api/resources/metadata_api.py b/lib/pbench/server/api/resources/metadata_api.py index 6641d31496..bb334f211f 100644 --- a/lib/pbench/server/api/resources/metadata_api.py +++ b/lib/pbench/server/api/resources/metadata_api.py @@ -4,6 +4,28 @@ from pbench.server.api.auth import Auth +def verify_metadata(metadata, logger): + current_user = Auth.token_auth.current_user() + metadata_user_id = metadata.user_id + if current_user is None: + # The request is not from a logged-in user + if metadata_user_id is None: + return True + logger.warning( + "Metadata user verification: Public user is trying to access private metadata object for user {}", + metadata_user_id, + ) + return False + if current_user.id != metadata_user_id and not current_user.is_admin(): + logger.warning( + "Metadata user verification: Logged in user_id {} is different than the one provided in the URI {}", + current_user.id, + metadata_user_id, + ) + return False + return True + + class CreateMetadata(Resource): """ Abstracted pbench API for handling user metadata @@ -14,7 +36,7 @@ def __init__(self, config, logger, auth): self.logger = logger self.auth = auth - @Auth.token_auth.login_required(optional=True) + @Auth.token_auth.login_required() def post(self): """ Post request for creating metadata instance for a user. @@ -27,9 +49,9 @@ def post(self): "value": "blog text" , } - Authorization header can be included as + Authorization token must be included in the header for creating metadata object Authorization: Bearer - If the authorization header is not present, the created metadata object becomes public by default + If the authorization header is not present, the user can not create the metadata objects on the server :return: JSON Payload response_object = { @@ -46,11 +68,7 @@ def post(self): self.logger.warning("Invalid json object: {}", request.url) abort(400, message="Invalid json object in request") - current_user = self.auth.token_auth.current_user() - if current_user: - current_user_id = current_user.id - else: - current_user_id = None + current_user_id = self.auth.token_auth.current_user().id value = post_data.get("value") if value is None: @@ -104,9 +122,12 @@ def __init__(self, config, logger, auth): def get(self, key): """ Get request for querying all the metadata objects for a user. - returns the list of all the metadata objects of a specified key associated with a logged in user. - If the user is not logged in we return all the public metadata objects of a specified key. - This requires a Pbench auth token in the header field + If the Pbench authorization token is provided in the header, + we return the list of all the metadata objects of a specified + key associated with a logged in user. + + If the authorization token is not provided in the header, + the access is restricted to only public metadata objects. Optional headers include Authorization: Bearer @@ -159,33 +180,13 @@ def __init__(self, config, logger): self.server_config = config self.logger = logger - def verify_metadata(self, metadata): - current_user = Auth.token_auth.current_user() - metadata_user_id = metadata.user_id - if current_user is None: - # The request is not from a logged-in user - if metadata_user_id is None: - return True - self.logger.warning( - "Metadata user verification: Public user is trying to access private metadata object for user {}", - metadata_user_id, - ) - return False - if current_user.id != metadata_user_id and not current_user.is_admin(): - self.logger.warning( - "Metadata user verification: Logged in user_id {} is different than the one provided in the URI {}", - current_user.id, - metadata_user_id, - ) - return False - return True - @Auth.token_auth.login_required(optional=True) def get(self, id): """ Get request for querying a metadata object of a user given a metadata id. This requires a Pbench auth token in the header field if the metadata object is private to a user + If the authorization token is not provided in the header, only public metadata objects are accessible The url requires a metadata object id such as /user/metadata/ @@ -223,8 +224,8 @@ def get(self, id): abort(404, message="Not found") # Verify if the metadata object id in the url belongs to the logged in user - if not self.verify_metadata(metadata_object): - abort(403, message="Not authorized to perform the GET request") + if not verify_metadata(metadata_object, self.logger): + abort(403, message="Not authorized to get the metadata object") response_object = { "data": metadata_object.get_json( @@ -233,11 +234,12 @@ def get(self, id): } return make_response(jsonify(response_object), 200) - @Auth.token_auth.login_required(optional=True) + @Auth.token_auth.login_required() def put(self, id): """ Put request for updating a metadata object of a user given a metadata id. - This requires a Pbench auth token in the header field + This requires a Pbench auth token in the header field. + Public metadata objects are read-only accept for an admin users. The url requires a metadata object id such as /user/metadata/ @@ -285,8 +287,8 @@ def put(self, id): abort(404, message="Not found") # Verify if the metadata object id in the url belongs to the logged in user - if not self.verify_metadata(metadata_object): - abort(403, message="Not authorized to perform the PUT request") + if not verify_metadata(metadata_object, self.logger): + abort(403, message="Not authorized to update the metadata object") # Check if the metadata payload contain fields that are either protected or # not present in the metadata db. If any key in the payload does not match @@ -323,11 +325,12 @@ def put(self, id): } return make_response(jsonify(response_object), 200) - @Auth.token_auth.login_required(optional=True) + @Auth.token_auth.login_required() def delete(self, id): """ Delete request for deleting a metadata object of a user given a metadata id. - This requires a Pbench auth token in the header field + This requires a Pbench auth token in the header field. + Public metadata objects can only be deleted by admin users. The url requires a metadata object id such as /user/metadata/ @@ -359,8 +362,8 @@ def delete(self, id): abort(404, message="Not found") # Verify if the metadata object id in the url belongs to the logged in user - if not self.verify_metadata(metadata_object): - abort(403, message="Not authorized to perform the DELETE request") + if not verify_metadata(metadata_object, self.logger): + abort(403, message="Not authorized to DELETE the metadata object") try: # Delete the metadata object @@ -376,3 +379,67 @@ def delete(self, id): "message": "Metadata object deleted", } return make_response(jsonify(response_object), 200) + + +class PublishMetadata(Resource): + """ + Abstracted pbench API for handling user metadata + """ + + def __init__(self, config, logger): + self.server_config = config + self.logger = logger + + @Auth.token_auth.login_required() + def post(self, id): + """ + Post request for publishing the metadata object for public access. + This requires a Pbench auth token in the header field. Only authorized users can make their metadata public. + Right now once the metadata object is made public it can not be reverted back to the private entity + TODO: Is it desirable to be able to revert the object back as a private entity + + Headers include + Authorization: Bearer + + :return: JSON Payload + response_object = { + "message": "Metadata object is published" + } + """ + if id is None: + self.logger.warning("Metadata id not provided during metadata query") + abort(400, message="Missing metadata id in the post request uri") + + try: + # Fetch the metadata object + metadata_objects = UserMetadata.query(id=id) + except Exception: + self.logger.exception( + "Exception occurred in the Delete request while querying the Metadata model, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + if metadata_objects: + metadata_object = metadata_objects[0] + else: + abort(404, message="Not found") + + # Verify if the metadata object id in the url belongs to the logged in user + if not verify_metadata(metadata_object, self.logger): + abort(403, message="Not authorized to publish the metadata object") + + try: + # Update the metadata object user_id to Null to make them public + metadata_object.update(**{"user_id": None}) + except Exception: + self.logger.exception( + "Exception occurred in the Delete request while querying the Metadata model, id: {}", + id, + ) + abort(500, message="INTERNAL ERROR") + + response_object = { + "message": "Metadata object is published", + } + return make_response(jsonify(response_object), 200) diff --git a/lib/pbench/test/unit/server/test_metadata_objects.py b/lib/pbench/test/unit/server/test_metadata_objects.py index a414b944e1..e4a88b038d 100644 --- a/lib/pbench/test/unit/server/test_metadata_objects.py +++ b/lib/pbench/test/unit/server/test_metadata_objects.py @@ -85,9 +85,7 @@ def test_unauthorized_metadata_creation(client, server_config): "value": '{"config": "config1", "description": "description1"}', }, ) - data = response.json - assert data - assert response.status_code == 201 + assert response.status_code == 401 # Create a favorite metadata object response = client.post( @@ -97,24 +95,19 @@ def test_unauthorized_metadata_creation(client, server_config): "value": '{"config": "config2", "description": "description2"}', }, ) - assert response.status_code == 201 - data = response.json - assert data["data"]["key"] == "favorite" + assert response.status_code == 401 - # Get all the favorite metadata objects of non-logged in user + # Get all the favorite metadata objects of non-logged in user, should not return any data response = client.get(f"{server_config.rest_uri}/metadata/favorite") assert response.status_code == 200 data = response.json - assert ( - data["data"][0]["value"] - == '{"config": "config2", "description": "description2"}' - ) + assert data["data"] == [] - # Get all the saved metadata objects of non-logged in user + # Get all the saved metadata objects of non-logged in user, should not return any data response = client.get(f"{server_config.rest_uri}/metadata/saved",) assert response.status_code == 200 data = response.json - assert len(data["data"]) == 1 + assert len(data["data"]) == 0 @staticmethod def test_single_metadata_query(client, server_config): @@ -212,7 +205,7 @@ def test_unauthorized_metadata_query2(client, server_config): headers=dict(Authorization="Bearer " + data_login_2["auth_token"]), ) data = response.json - assert data["message"] == "Not authorized to perform the GET request" + assert data["message"] == "Not authorized to get the metadata object" assert response.status_code == 403 @staticmethod @@ -289,3 +282,33 @@ def test_metadata_delete(client, server_config): data = response.json assert data["message"] == "Metadata object deleted" assert response.status_code == 200 + + @staticmethod + def test_publish_metadata_object(client, server_config): + data_login = user_register_login(client, server_config) + with client: + response = client.post( + f"{server_config.rest_uri}/metadata", + json={ + "key": "favorite", + "value": '{"config": "config1", "description": "description2"}', + }, + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["data"]["id"] + + metadata_id = data["data"]["id"] + response = client.post( + f"{server_config.rest_uri}/metadata/{metadata_id}/publish", + headers=dict(Authorization="Bearer " + data_login["auth_token"]), + ) + data = response.json + assert data["message"] == "Metadata object is published" + assert response.status_code == 200 + + # Test if non logged-in user can access this data now + response = client.get(f"{server_config.rest_uri}/metadata/favorite") + assert response.status_code == 200 + data = response.json + assert len(data["data"]) == 1