diff --git a/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java new file mode 100644 index 000000000000..8c7c4ef16c3f --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.catalog; + +import java.util.List; +import java.util.Map; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchViewException; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; + +/** A session Catalog API for view create, drop, and load operations. */ +public interface ViewSessionCatalog { + + /** + * Return the name for this catalog. + * + * @return this catalog's name + */ + String name(); + + /** + * Return all the identifiers under this namespace. + * + * @param namespace a namespace + * @return a list of identifiers for views + * @throws NoSuchNamespaceException if the namespace is not found + */ + List listViews(SessionCatalog.SessionContext context, Namespace namespace); + + /** + * Load a view. + * + * @param identifier a view identifier + * @return instance of {@link View} implementation referred by the identifier + * @throws NoSuchViewException if the view does not exist + */ + View loadView(SessionCatalog.SessionContext context, TableIdentifier identifier); + + /** + * Check whether view exists. + * + * @param identifier a view identifier + * @return true if the view exists, false otherwise + */ + default boolean viewExists(SessionCatalog.SessionContext context, TableIdentifier identifier) { + try { + loadView(context, identifier); + return true; + } catch (NoSuchViewException e) { + return false; + } + } + + /** + * Instantiate a builder to create or replace a SQL view. + * + * @param identifier a view identifier + * @return a view builder + */ + ViewBuilder buildView(SessionCatalog.SessionContext context, TableIdentifier identifier); + + /** + * Drop a view. + * + * @param identifier a view identifier + * @return true if the view was dropped, false if the view did not exist + */ + boolean dropView(SessionCatalog.SessionContext context, TableIdentifier identifier); + + /** + * Rename a view. + * + * @param from identifier of the view to rename + * @param to new view identifier + * @throws NoSuchViewException if the "from" view does not exist + * @throws AlreadyExistsException if the "to" view already exists + */ + void renameView(SessionCatalog.SessionContext context, TableIdentifier from, TableIdentifier to); + + /** + * Invalidate cached view metadata from current catalog. + * + *

If the view is already loaded or cached, drop cached data. If the view does not exist or is + * not cached, do nothing. + * + * @param identifier a view identifier + */ + default void invalidateView(SessionCatalog.SessionContext context, TableIdentifier identifier) {} + + /** + * Initialize a view catalog given a custom name and a map of catalog properties. + * + *

A custom view catalog implementation must have a no-arg constructor. A compute engine like + * Spark or Flink will first initialize the catalog without any arguments, and then call this + * method to complete catalog initialization with properties passed into the engine. + * + * @param name a custom name for the catalog + * @param properties catalog properties + */ + default void initialize(String name, Map properties) {} +} diff --git a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java index d6ee4d345cfa..874d0ec6cab9 100644 --- a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java @@ -30,8 +30,10 @@ import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; -public abstract class BaseSessionCatalog implements SessionCatalog { +public abstract class BaseSessionCatalog implements SessionCatalog, ViewSessionCatalog { private final Cache catalogs = Caffeine.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES).build(); @@ -62,7 +64,7 @@ public T withContext(SessionContext context, Function task) { return task.apply(asCatalog(context)); } - public class AsCatalog implements Catalog, SupportsNamespaces { + public class AsCatalog implements Catalog, SupportsNamespaces, ViewCatalog { private final SessionContext context; private AsCatalog(SessionContext context) { @@ -84,6 +86,11 @@ public TableBuilder buildTable(TableIdentifier ident, Schema schema) { return BaseSessionCatalog.this.buildTable(context, ident, schema); } + @Override + public void initialize(String catalogName, Map props) { + BaseSessionCatalog.this.initialize(catalogName, props); + } + @Override public Table registerTable(TableIdentifier ident, String metadataFileLocation) { return BaseSessionCatalog.this.registerTable(context, ident, metadataFileLocation); @@ -159,5 +166,40 @@ public boolean removeProperties(Namespace namespace, Set removals) { public boolean namespaceExists(Namespace namespace) { return BaseSessionCatalog.this.namespaceExists(context, namespace); } + + @Override + public List listViews(Namespace namespace) { + return BaseSessionCatalog.this.listViews(context, namespace); + } + + @Override + public View loadView(TableIdentifier identifier) { + return BaseSessionCatalog.this.loadView(context, identifier); + } + + @Override + public boolean viewExists(TableIdentifier identifier) { + return BaseSessionCatalog.this.viewExists(context, identifier); + } + + @Override + public ViewBuilder buildView(TableIdentifier identifier) { + return BaseSessionCatalog.this.buildView(context, identifier); + } + + @Override + public boolean dropView(TableIdentifier identifier) { + return BaseSessionCatalog.this.dropView(context, identifier); + } + + @Override + public void renameView(TableIdentifier from, TableIdentifier to) { + BaseSessionCatalog.this.renameView(context, from, to); + } + + @Override + public void invalidateView(TableIdentifier identifier) { + BaseSessionCatalog.this.invalidateView(context, identifier); + } } } diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 1e0ef660271b..8f051a0ad129 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -45,26 +45,37 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.rest.responses.CreateNamespaceResponse; import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ImmutableLoadViewResponse; import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.SQLViewRepresentation; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.iceberg.view.ViewOperations; public class CatalogHandlers { private static final Schema EMPTY_SCHEMA = new Schema(); @@ -374,4 +385,107 @@ static TableMetadata commit(TableOperations ops, UpdateTableRequest request) { return ops.current(); } + + public static BaseView baseView(View view) { + Preconditions.checkArgument(view instanceof BaseView, "View must be a BaseView"); + return (BaseView) view; + } + + public static ListTablesResponse listViews(ViewCatalog catalog, Namespace namespace) { + return ListTablesResponse.builder().addAll(catalog.listViews(namespace)).build(); + } + + public static LoadViewResponse createView( + ViewCatalog catalog, Namespace namespace, CreateViewRequest request) { + request.validate(); + + ViewMetadata viewMetadata = request.metadata(); + ViewBuilder viewBuilder = + catalog + .buildView(TableIdentifier.of(namespace, request.name())) + .withSchema(viewMetadata.schema()) + .withProperties(viewMetadata.properties()) + .withDefaultNamespace(viewMetadata.currentVersion().defaultNamespace()) + .withDefaultCatalog(viewMetadata.currentVersion().defaultCatalog()); + viewMetadata.currentVersion().representations().stream() + .filter(r -> r instanceof SQLViewRepresentation) + .map(r -> (SQLViewRepresentation) r) + .forEach(r -> viewBuilder.withQuery(r.dialect(), r.sql())); + View view = viewBuilder.create(); + + return viewResponse(view); + } + + private static LoadViewResponse viewResponse(View view) { + ViewMetadata metadata = baseView(view).operations().current(); + return ImmutableLoadViewResponse.builder() + .metadata(metadata) + .metadataLocation(metadata.metadataFileLocation()) + .build(); + } + + public static LoadViewResponse loadView(ViewCatalog catalog, TableIdentifier viewIdentifier) { + View view = catalog.loadView(viewIdentifier); + return viewResponse(view); + } + + public static LoadViewResponse updateView( + ViewCatalog catalog, TableIdentifier ident, UpdateTableRequest request) { + View view = catalog.loadView(ident); + ViewMetadata metadata = commit(baseView(view).operations(), request); + + return ImmutableLoadViewResponse.builder() + .metadata(metadata) + .metadataLocation(metadata.metadataFileLocation()) + .build(); + } + + public static void renameView(ViewCatalog catalog, RenameTableRequest request) { + catalog.renameView(request.source(), request.destination()); + } + + public static void dropView(ViewCatalog catalog, TableIdentifier viewIdentifier) { + boolean dropped = catalog.dropView(viewIdentifier); + if (!dropped) { + throw new NoSuchViewException("View does not exist: %s", viewIdentifier); + } + } + + static ViewMetadata commit(ViewOperations ops, UpdateTableRequest request) { + AtomicBoolean isRetry = new AtomicBoolean(false); + try { + Tasks.foreach(ops) + .retry(COMMIT_NUM_RETRIES_DEFAULT) + .exponentialBackoff( + COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + 2.0 /* exponential */) + .onlyRetryOn(CommitFailedException.class) + .run( + taskOps -> { + ViewMetadata base = isRetry.get() ? taskOps.refresh() : taskOps.current(); + isRetry.set(true); + + // apply changes + ViewMetadata.Builder metadataBuilder = ViewMetadata.buildFrom(base); + request.updates().forEach(update -> update.applyTo(metadataBuilder)); + + ViewMetadata updated = metadataBuilder.build(); + + if (updated.changes().isEmpty()) { + // do not commit if the metadata has not changed + return; + } + + // commit + taskOps.commit(base, updated); + }); + + } catch (ValidationFailureException e) { + throw e.wrapped(); + } + + return ops.current(); + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java index 5a7f9b59f2d4..846820a99d9f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java @@ -26,6 +26,7 @@ import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.iceberg.exceptions.RESTException; import org.apache.iceberg.exceptions.ServiceFailureException; @@ -55,6 +56,14 @@ public static Consumer tableErrorHandler() { return TableErrorHandler.INSTANCE; } + public static Consumer viewErrorHandler() { + return ViewErrorHandler.INSTANCE; + } + + public static Consumer viewCommitHandler() { + return ViewCommitErrorHandler.INSTANCE; + } + public static Consumer tableCommitHandler() { return CommitErrorHandler.INSTANCE; } @@ -110,6 +119,49 @@ public void accept(ErrorResponse error) { } } + /** View commit error handler. */ + private static class ViewCommitErrorHandler extends DefaultErrorHandler { + private static final ErrorHandler INSTANCE = new ViewCommitErrorHandler(); + + @Override + public void accept(ErrorResponse error) { + switch (error.code()) { + case 404: + throw new NoSuchViewException("%s", error.message()); + case 409: + throw new CommitFailedException("Commit failed: %s", error.message()); + case 500: + case 502: + case 504: + throw new CommitStateUnknownException( + new ServiceFailureException("Service failed: %s: %s", error.code(), error.message())); + } + + super.accept(error); + } + } + + /** View level error handler. */ + private static class ViewErrorHandler extends DefaultErrorHandler { + private static final ErrorHandler INSTANCE = new ViewErrorHandler(); + + @Override + public void accept(ErrorResponse error) { + switch (error.code()) { + case 404: + if (NoSuchNamespaceException.class.getSimpleName().equals(error.type())) { + throw new NoSuchNamespaceException("%s", error.message()); + } else { + throw new NoSuchViewException("%s", error.message()); + } + case 409: + throw new AlreadyExistsException("%s", error.message()); + } + + super.accept(error); + } + } + /** Request error handler specifically for CRUD ops on namespaces. */ private static class NamespaceErrorHandler extends DefaultErrorHandler { private static final ErrorHandler INSTANCE = new NamespaceErrorHandler(); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java index 63b660c46aa3..b50ac79d3379 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java @@ -35,17 +35,22 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableCommit; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; -public class RESTCatalog implements Catalog, SupportsNamespaces, Configurable, Closeable { +public class RESTCatalog + implements Catalog, SupportsNamespaces, Configurable, Closeable, ViewCatalog { private final RESTSessionCatalog sessionCatalog; private final Catalog delegate; private final SupportsNamespaces nsDelegate; private final SessionCatalog.SessionContext context; + private final ViewCatalog viewDelegate; public RESTCatalog() { this( @@ -64,6 +69,7 @@ public RESTCatalog( this.delegate = sessionCatalog.asCatalog(context); this.nsDelegate = (SupportsNamespaces) delegate; this.context = context; + this.viewDelegate = (ViewCatalog) delegate; } @Override @@ -261,4 +267,39 @@ public void commitTransaction(TableCommit... commits) { sessionCatalog.commitTransaction( context, ImmutableList.builder().add(commits).build()); } + + @Override + public List listViews(Namespace namespace) { + return viewDelegate.listViews(namespace); + } + + @Override + public View loadView(TableIdentifier identifier) { + return viewDelegate.loadView(identifier); + } + + @Override + public ViewBuilder buildView(TableIdentifier identifier) { + return viewDelegate.buildView(identifier); + } + + @Override + public boolean dropView(TableIdentifier identifier) { + return viewDelegate.dropView(identifier); + } + + @Override + public void renameView(TableIdentifier from, TableIdentifier to) { + viewDelegate.renameView(from, to); + } + + @Override + public boolean viewExists(TableIdentifier identifier) { + return viewDelegate.viewExists(identifier); + } + + @Override + public void invalidateView(TableIdentifier identifier) { + viewDelegate.invalidateView(identifier); + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java b/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java index 06f2de32df0c..7d4f327b67df 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java @@ -44,6 +44,9 @@ import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CommitTransactionRequestParser; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.CreateViewRequestParser; +import org.apache.iceberg.rest.requests.ImmutableCreateViewRequest; import org.apache.iceberg.rest.requests.ImmutableRegisterTableRequest; import org.apache.iceberg.rest.requests.ImmutableReportMetricsRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; @@ -56,6 +59,9 @@ import org.apache.iceberg.rest.requests.UpdateTableRequestParser; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.rest.responses.ErrorResponseParser; +import org.apache.iceberg.rest.responses.ImmutableLoadViewResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.rest.responses.LoadViewResponseParser; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.util.JsonUtil; @@ -101,7 +107,16 @@ public static void registerAll(ObjectMapper mapper) { .addDeserializer(RegisterTableRequest.class, new RegisterTableRequestDeserializer<>()) .addSerializer(ImmutableRegisterTableRequest.class, new RegisterTableRequestSerializer<>()) .addDeserializer( - ImmutableRegisterTableRequest.class, new RegisterTableRequestDeserializer<>()); + ImmutableRegisterTableRequest.class, new RegisterTableRequestDeserializer<>()) + .addSerializer(CreateViewRequest.class, new CreateViewRequestSerializer<>()) + .addSerializer(ImmutableCreateViewRequest.class, new CreateViewRequestSerializer<>()) + .addDeserializer(CreateViewRequest.class, new CreateViewRequestDeserializer<>()) + .addDeserializer(ImmutableCreateViewRequest.class, new CreateViewRequestDeserializer<>()) + .addSerializer(LoadViewResponse.class, new LoadViewResponseSerializer<>()) + .addSerializer(ImmutableLoadViewResponse.class, new LoadViewResponseSerializer<>()) + .addDeserializer(LoadViewResponse.class, new LoadViewResponseDeserializer<>()) + .addDeserializer(ImmutableLoadViewResponse.class, new LoadViewResponseDeserializer<>()); + mapper.registerModule(module); } @@ -379,4 +394,38 @@ public T deserialize(JsonParser p, DeserializationContext context) throws IOExce return (T) RegisterTableRequestParser.fromJson(jsonNode); } } + + static class CreateViewRequestSerializer extends JsonSerializer { + @Override + public void serialize(T request, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + CreateViewRequestParser.toJson(request, gen); + } + } + + static class CreateViewRequestDeserializer + extends JsonDeserializer { + @Override + public T deserialize(JsonParser p, DeserializationContext context) throws IOException { + JsonNode jsonNode = p.getCodec().readTree(p); + return (T) CreateViewRequestParser.fromJson(jsonNode); + } + } + + static class LoadViewResponseSerializer extends JsonSerializer { + @Override + public void serialize(T request, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + LoadViewResponseParser.toJson(request, gen); + } + } + + static class LoadViewResponseDeserializer + extends JsonDeserializer { + @Override + public T deserialize(JsonParser p, DeserializationContext context) throws IOException { + JsonNode jsonNode = p.getCodec().readTree(p); + return (T) LoadViewResponseParser.fromJson(jsonNode); + } + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 72547eec3486..4eea7c6823ab 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -46,7 +46,6 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableOperations; import org.apache.iceberg.Transaction; import org.apache.iceberg.Transactions; import org.apache.iceberg.catalog.BaseSessionCatalog; @@ -56,6 +55,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; @@ -65,12 +65,15 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.rest.auth.OAuth2Util.AuthSession; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.ImmutableCreateViewRequest; import org.apache.iceberg.rest.requests.ImmutableRegisterTableRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; @@ -82,11 +85,21 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.EnvironmentUtil; import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.util.ThreadPools; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.ImmutableSQLViewRepresentation; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.iceberg.view.ViewRepresentation; +import org.apache.iceberg.view.ViewUtil; +import org.apache.iceberg.view.ViewVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,7 +120,7 @@ public class RESTSessionCatalog extends BaseSessionCatalog private final Function, RESTClient> clientBuilder; private final BiFunction, FileIO> ioBuilder; private Cache sessions = null; - private Cache fileIOCloser; + private Cache fileIOCloser; private AuthSession catalogAuth = null; private boolean keepTokenRefreshed = true; private RESTClient client = null; @@ -383,6 +396,12 @@ private void trackFileIO(RESTTableOperations ops) { } } + private void trackFileIO(RESTViewOperations ops) { + if (io != ops.io()) { + fileIOCloser.put(ops, ops.io()); + } + } + private MetricsReporter metricsReporter( String metricsEndpoint, Supplier> headers) { if (reportingViaRestEnabled) { @@ -919,6 +938,12 @@ private void checkIdentifierIsValid(TableIdentifier tableIdentifier) { } } + private void checkViewIdentifierIsValid(TableIdentifier identifier) { + if (identifier.namespace().isEmpty()) { + throw new NoSuchViewException("Invalid view identifier: %s", identifier); + } + } + private void checkNamespaceIsValid(Namespace namespace) { if (namespace.isEmpty()) { throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); @@ -943,11 +968,11 @@ private static Cache newSessionCache(Map pr .build(); } - private Cache newFileIOCloser() { + private Cache newFileIOCloser() { return Caffeine.newBuilder() .weakKeys() .removalListener( - (RemovalListener) + (RemovalListener) (ops, fileIO, cause) -> { if (null != fileIO) { fileIO.close(); @@ -971,4 +996,229 @@ public void commitTransaction(SessionContext context, List commits) headers(context), ErrorHandlers.tableCommitHandler()); } + + @Override + public List listViews(SessionContext context, Namespace namespace) { + checkNamespaceIsValid(namespace); + + ListTablesResponse response = + client.get( + paths.views(namespace), + ListTablesResponse.class, + headers(context), + ErrorHandlers.namespaceErrorHandler()); + return response.identifiers(); + } + + @Override + public View loadView(SessionContext context, TableIdentifier identifier) { + checkViewIdentifierIsValid(identifier); + + LoadViewResponse response = + client.get( + paths.view(identifier), + LoadViewResponse.class, + headers(context), + ErrorHandlers.viewErrorHandler()); + + AuthSession session = tableSession(response.config(), session(context)); + ViewMetadata metadata = response.metadata(); + + RESTViewOperations ops = + new RESTViewOperations( + client, + paths.view(identifier), + session::headers, + tableFileIO(context, response.config()), + metadata); + + trackFileIO(ops); + + return new BaseView(ops, ViewUtil.fullViewName(name(), identifier)); + } + + @Override + public RESTViewBuilder buildView(SessionContext context, TableIdentifier identifier) { + return new RESTViewBuilder(context, identifier); + } + + @Override + public boolean dropView(SessionContext context, TableIdentifier identifier) { + checkViewIdentifierIsValid(identifier); + + try { + client.delete( + paths.view(identifier), null, headers(context), ErrorHandlers.viewErrorHandler()); + return true; + } catch (NoSuchViewException e) { + return false; + } + } + + @Override + public void renameView(SessionContext context, TableIdentifier from, TableIdentifier to) { + checkViewIdentifierIsValid(from); + checkViewIdentifierIsValid(to); + + RenameTableRequest request = + RenameTableRequest.builder().withSource(from).withDestination(to).build(); + + client.post( + paths.renameView(), request, null, headers(context), ErrorHandlers.viewErrorHandler()); + } + + private class RESTViewBuilder implements ViewBuilder { + private final SessionContext context; + private final TableIdentifier identifier; + private final ImmutableViewVersion.Builder viewVersionBuilder = ImmutableViewVersion.builder(); + private final List viewRepresentations = Lists.newArrayList(); + private final Map properties = Maps.newHashMap(); + private Schema schema; + + private RESTViewBuilder(SessionContext context, TableIdentifier identifier) { + checkViewIdentifierIsValid(identifier); + this.identifier = identifier; + this.context = context; + } + + @Override + public ViewBuilder withSchema(Schema newSchema) { + this.schema = newSchema; + viewVersionBuilder.schemaId(newSchema.schemaId()); + return this; + } + + @Override + public ViewBuilder withQuery(String dialect, String sql) { + viewRepresentations.add( + ImmutableSQLViewRepresentation.builder().dialect(dialect).sql(sql).build()); + return this; + } + + @Override + public ViewBuilder withDefaultCatalog(String defaultCatalog) { + viewVersionBuilder.defaultCatalog(defaultCatalog); + return this; + } + + @Override + public ViewBuilder withDefaultNamespace(Namespace namespace) { + viewVersionBuilder.defaultNamespace(namespace); + return this; + } + + @Override + public ViewBuilder withProperties(Map newProperties) { + this.properties.putAll(newProperties); + return this; + } + + @Override + public ViewBuilder withProperty(String key, String value) { + this.properties.put(key, value); + return this; + } + + @Override + public View create() { + ViewVersion viewVersion = + viewVersionBuilder + .versionId(1) + .addAllRepresentations(viewRepresentations) + .timestampMillis(System.currentTimeMillis()) + .putSummary("operation", "create") + .build(); + + ViewMetadata viewMetadata = + ViewMetadata.builder() + .setProperties(properties) + .setLocation("dummy") + .setCurrentVersion(viewVersion, schema) + .build(); + + CreateViewRequest request = + ImmutableCreateViewRequest.builder() + .metadata(viewMetadata) + .name(identifier.name()) + .build(); + + LoadViewResponse response = + client.post( + paths.views(identifier.namespace()), + request, + LoadViewResponse.class, + headers(context), + ErrorHandlers.viewErrorHandler()); + + return viewFromResponse(response); + } + + @Override + public View replace() { + LoadViewResponse response = + client.get( + paths.view(identifier), + LoadViewResponse.class, + headers(context), + ErrorHandlers.viewErrorHandler()); + + ViewMetadata metadata = response.metadata(); + + ViewVersion viewVersion = + viewVersionBuilder + .versionId(metadata.currentVersionId() + 1) + .addAllRepresentations(viewRepresentations) + .timestampMillis(System.currentTimeMillis()) + .putSummary("operation", "replace") + .build(); + + ViewMetadata replacement = + ViewMetadata.buildFrom(metadata) + .setProperties(properties) + .setCurrentVersion(viewVersion, schema) + .build(); + + UpdateTableRequest request = + UpdateTableRequest.create(identifier, ImmutableList.of(), replacement.changes()); + + response = + client.post( + paths.view(identifier), + request, + LoadViewResponse.class, + headers(context), + ErrorHandlers.viewErrorHandler()); + + return viewFromResponse(response); + } + + private BaseView viewFromResponse(LoadViewResponse response) { + AuthSession session = tableSession(response.config(), session(context)); + RESTViewOperations ops = + new RESTViewOperations( + client, + paths.view(identifier), + session::headers, + tableFileIO(context, response.config()), + response.metadata()); + + trackFileIO(ops); + + return new BaseView(ops, ViewUtil.fullViewName(name(), identifier)); + } + + @Override + public View createOrReplace() { + try { + client.get( + paths.view(identifier), + LoadViewResponse.class, + headers(context), + ErrorHandlers.viewErrorHandler()); + return replace(); + } catch (NoSuchViewException e) { + return create(); + } + } + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTViewOperations.java b/core/src/main/java/org/apache/iceberg/rest/RESTViewOperations.java new file mode 100644 index 000000000000..930295904e13 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/RESTViewOperations.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.iceberg.view.ViewOperations; + +class RESTViewOperations implements ViewOperations { + enum UpdateType { + CREATE, + REPLACE + } + + private final RESTClient client; + private final String path; + private final Supplier> headers; + private final List createChanges; + private UpdateType updateType; + private final FileIO io; + private ViewMetadata current; + + RESTViewOperations( + RESTClient client, + String path, + Supplier> headers, + FileIO io, + ViewMetadata current) { + this(client, path, headers, io, ImmutableList.of(), UpdateType.REPLACE, current); + } + + RESTViewOperations( + RESTClient client, + String path, + Supplier> headers, + FileIO io, + List createChanges, + UpdateType updateType, + ViewMetadata current) { + this.client = client; + this.path = path; + this.headers = headers; + this.io = io; + this.createChanges = createChanges; + this.updateType = updateType; + if (updateType == UpdateType.CREATE) { + this.current = null; + } else { + this.current = current; + } + } + + @Override + public ViewMetadata current() { + return current; + } + + @Override + public ViewMetadata refresh() { + return updateCurrentMetadata( + client.get(path, LoadViewResponse.class, headers, ErrorHandlers.viewErrorHandler())); + } + + @Override + public void commit(ViewMetadata base, ViewMetadata metadata) { + List updates = + ImmutableList.builder() + .addAll(createChanges) + .addAll(metadata.changes()) + .build(); + + if (UpdateType.CREATE == updateType) { + Preconditions.checkState( + base == null, "Invalid base metadata for create, expected null: %s", base); + } else { + Preconditions.checkState(base != null, "Invalid base metadata: null"); + } + + UpdateTableRequest request = UpdateTableRequest.create(null, ImmutableList.of(), updates); + + LoadViewResponse response = + client.post( + path, request, LoadViewResponse.class, headers, ErrorHandlers.viewCommitHandler()); + + // all future commits should replace the view + this.updateType = UpdateType.REPLACE; + + updateCurrentMetadata(response); + } + + public FileIO io() { + return io; + } + + private ViewMetadata updateCurrentMetadata(LoadViewResponse response) { + if (current == null || !Objects.equals(current.location(), response.metadataLocation())) { + this.current = response.metadata(); + } + + return current; + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java index b5b974f14c5e..c68a4f450843 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java +++ b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java @@ -93,4 +93,22 @@ public String metrics(TableIdentifier identifier) { public String commitTransaction() { return SLASH.join("v1", prefix, "transactions", "commit"); } + + public String views(Namespace ns) { + return SLASH.join("v1", prefix, "namespaces", RESTUtil.encodeNamespace(ns), "views"); + } + + public String view(TableIdentifier ident) { + return SLASH.join( + "v1", + prefix, + "namespaces", + RESTUtil.encodeNamespace(ident.namespace()), + "views", + RESTUtil.encodeString(ident.name())); + } + + public String renameView() { + return SLASH.join("v1", prefix, "views", "rename"); + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequest.java b/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequest.java new file mode 100644 index 000000000000..723dd936413d --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.requests; + +import org.apache.iceberg.rest.RESTRequest; +import org.apache.iceberg.view.ViewMetadata; +import org.immutables.value.Value; + +@Value.Immutable +public interface CreateViewRequest extends RESTRequest { + String name(); + + ViewMetadata metadata(); + + @Override + default void validate() { + // nothing to validate as it's not possible to create an invalid instance + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequestParser.java b/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequestParser.java new file mode 100644 index 000000000000..48e95017d206 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/requests/CreateViewRequestParser.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.requests; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; +import org.apache.iceberg.view.ViewMetadataParser; + +public class CreateViewRequestParser { + + private static final String NAME = "name"; + private static final String METADATA = "metadata"; + + private CreateViewRequestParser() {} + + public static String toJson(CreateViewRequest request) { + return toJson(request, false); + } + + public static String toJson(CreateViewRequest request, boolean pretty) { + return JsonUtil.generate(gen -> toJson(request, gen), pretty); + } + + public static void toJson(CreateViewRequest request, JsonGenerator gen) throws IOException { + Preconditions.checkArgument(null != request, "Invalid create view request: null"); + + gen.writeStartObject(); + + gen.writeStringField(NAME, request.name()); + + gen.writeFieldName(METADATA); + ViewMetadataParser.toJson(request.metadata(), gen); + + gen.writeEndObject(); + } + + public static CreateViewRequest fromJson(String json) { + return JsonUtil.parse(json, CreateViewRequestParser::fromJson); + } + + public static CreateViewRequest fromJson(JsonNode json) { + Preconditions.checkArgument(null != json, "Cannot parse create view request from null object"); + + return ImmutableCreateViewRequest.builder() + .name(JsonUtil.getString(NAME, json)) + .metadata(ViewMetadataParser.fromJson(JsonUtil.get(METADATA, json))) + .build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/RenameTableRequest.java b/core/src/main/java/org/apache/iceberg/rest/requests/RenameTableRequest.java index bb44410f2bc6..fad0ea3cd236 100644 --- a/core/src/main/java/org/apache/iceberg/rest/requests/RenameTableRequest.java +++ b/core/src/main/java/org/apache/iceberg/rest/requests/RenameTableRequest.java @@ -23,7 +23,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.rest.RESTRequest; -/** A REST request to rename a table. */ +/** A REST request to rename a table or a view. */ public class RenameTableRequest implements RESTRequest { private TableIdentifier source; diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java new file mode 100644 index 000000000000..d07ba872fdaa --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import java.util.Map; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.view.ViewMetadata; +import org.immutables.value.Value; + +@Value.Immutable +public interface LoadViewResponse extends RESTResponse { + String metadataLocation(); + + ViewMetadata metadata(); + + Map config(); + + @Override + default void validate() { + // nothing to validate as it's not possible to create an invalid instance + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java new file mode 100644 index 000000000000..a683d05acb78 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; +import org.apache.iceberg.view.ViewMetadataParser; + +public class LoadViewResponseParser { + + private static final String METADATA_LOCATION = "metadata-location"; + private static final String METADATA = "metadata"; + private static final String CONFIG = "config"; + + private LoadViewResponseParser() {} + + public static String toJson(LoadViewResponse response) { + return toJson(response, false); + } + + public static String toJson(LoadViewResponse response, boolean pretty) { + return JsonUtil.generate(gen -> toJson(response, gen), pretty); + } + + public static void toJson(LoadViewResponse response, JsonGenerator gen) throws IOException { + Preconditions.checkArgument(null != response, "Invalid load view response: null"); + + gen.writeStartObject(); + + gen.writeStringField(METADATA_LOCATION, response.metadataLocation()); + + gen.writeFieldName(METADATA); + ViewMetadataParser.toJson(response.metadata(), gen); + + if (!response.config().isEmpty()) { + JsonUtil.writeStringMap(CONFIG, response.config(), gen); + } + + gen.writeEndObject(); + } + + public static LoadViewResponse fromJson(String json) { + return JsonUtil.parse(json, LoadViewResponseParser::fromJson); + } + + public static LoadViewResponse fromJson(JsonNode json) { + Preconditions.checkArgument(null != json, "Cannot parse load view response from null object"); + + ImmutableLoadViewResponse.Builder builder = + ImmutableLoadViewResponse.builder() + .metadataLocation(JsonUtil.getString(METADATA_LOCATION, json)) + .metadata(ViewMetadataParser.fromJson(JsonUtil.get(METADATA, json))); + + if (json.has(CONFIG)) { + builder.config(JsonUtil.getStringMap(CONFIG, json)); + } + + return builder.build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java index 870deb801964..32446e0a124f 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java @@ -125,13 +125,6 @@ default void check() { formatVersion() > 0 && formatVersion() <= ViewMetadata.SUPPORTED_VIEW_FORMAT_VERSION, "Unsupported format version: %s", formatVersion()); - - // when associated with a metadata file, metadata must have no changes so that the metadata - // matches exactly what is in the metadata file, which does not store changes. metadata location - // with changes is inconsistent. - Preconditions.checkArgument( - metadataFileLocation() == null || changes().isEmpty(), - "Cannot create view metadata with a metadata location and changes"); } static Builder builder() { @@ -182,7 +175,6 @@ private Builder(ViewMetadata base) { this.formatVersion = base.formatVersion(); this.currentVersionId = base.currentVersionId(); this.location = base.location(); - this.metadataLocation = base.metadataFileLocation(); } public Builder upgradeFormatVersion(int newFormatVersion) { @@ -375,6 +367,13 @@ public ViewMetadata build() { Preconditions.checkArgument(null != location, "Invalid location: null"); Preconditions.checkArgument(versions.size() > 0, "Invalid view: no versions were added"); + // when associated with a metadata file, metadata must have no changes so that the metadata + // matches exactly what is in the metadata file, which does not store changes. metadata + // location with changes is inconsistent. + Preconditions.checkArgument( + metadataLocation == null || changes.isEmpty(), + "Cannot create view metadata with a metadata location and changes"); + int historySize = PropertyUtil.propertyAsInt( properties, diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java index a88369574c6b..70e86815774b 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java @@ -57,7 +57,7 @@ public static String toJson(ViewMetadata metadata, boolean pretty) { return JsonUtil.generate(gen -> toJson(metadata, gen), pretty); } - static void toJson(ViewMetadata metadata, JsonGenerator gen) throws IOException { + public static void toJson(ViewMetadata metadata, JsonGenerator gen) throws IOException { Preconditions.checkArgument(null != metadata, "Invalid view metadata: null"); gen.writeStartObject(); @@ -160,7 +160,7 @@ public static void write(ViewMetadata metadata, OutputFile outputFile) { public static ViewMetadata read(InputFile file) { try (InputStream is = file.newStream()) { - return fromJson(JsonUtil.mapper().readValue(is, JsonNode.class)); + return fromJson(file.location(), JsonUtil.mapper().readValue(is, JsonNode.class)); } catch (IOException e) { throw new UncheckedIOException(String.format("Failed to read json file: %s", file), e); } diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java index 0772601b77df..da8ccc091c5e 100644 --- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java +++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java @@ -31,6 +31,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; @@ -39,6 +40,7 @@ import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotAuthorizedException; import org.apache.iceberg.exceptions.RESTException; import org.apache.iceberg.exceptions.UnprocessableEntityException; @@ -49,6 +51,7 @@ import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.ReportMetricsRequest; @@ -61,6 +64,7 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Pair; @@ -79,6 +83,7 @@ public class RESTCatalogAdapter implements RESTClient { .put(ForbiddenException.class, 403) .put(NoSuchNamespaceException.class, 404) .put(NoSuchTableException.class, 404) + .put(NoSuchViewException.class, 404) .put(NoSuchIcebergTableException.class, 404) .put(UnsupportedOperationException.class, 406) .put(AlreadyExistsException.class, 409) @@ -89,11 +94,13 @@ public class RESTCatalogAdapter implements RESTClient { private final Catalog catalog; private final SupportsNamespaces asNamespaceCatalog; + private final ViewCatalog viewCatalog; public RESTCatalogAdapter(Catalog catalog) { this.catalog = catalog; this.asNamespaceCatalog = catalog instanceof SupportsNamespaces ? (SupportsNamespaces) catalog : null; + this.viewCatalog = catalog instanceof ViewCatalog ? (ViewCatalog) catalog : null; } enum HTTPMethod { @@ -145,7 +152,22 @@ enum Route { ReportMetricsRequest.class, null), COMMIT_TRANSACTION( - HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null); + HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null), + LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views", null, ListTablesResponse.class), + LOAD_VIEW( + HTTPMethod.GET, "v1/namespaces/{namespace}/views/{view}", null, LoadViewResponse.class), + CREATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views", + CreateViewRequest.class, + LoadViewResponse.class), + UPDATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views/{view}", + UpdateTableRequest.class, + LoadViewResponse.class), + RENAME_VIEW(HTTPMethod.POST, "v1/views/rename", RenameTableRequest.class, null), + DROP_VIEW(HTTPMethod.DELETE, "v1/namespaces/{namespace}/views/{view}"); private final HTTPMethod method; private final int requiredLength; @@ -223,7 +245,7 @@ public Class responseClass() { } } - @SuppressWarnings("MethodLength") + @SuppressWarnings({"MethodLength", "checkstyle:CyclomaticComplexity"}) public T handleRequest( Route route, Map vars, Object body, Class responseType) { switch (route) { @@ -387,6 +409,65 @@ public T handleRequest( return null; } + case LIST_VIEWS: + { + if (null != viewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.listViews(viewCatalog, namespace)); + } + break; + } + + case CREATE_VIEW: + { + if (null != viewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + CreateViewRequest request = castRequest(CreateViewRequest.class, body); + return castResponse( + responseType, CatalogHandlers.createView(viewCatalog, namespace, request)); + } + break; + } + + case LOAD_VIEW: + { + if (null != viewCatalog) { + TableIdentifier ident = viewIdentFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.loadView(viewCatalog, ident)); + } + break; + } + + case UPDATE_VIEW: + { + if (null != viewCatalog) { + TableIdentifier ident = viewIdentFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse( + responseType, CatalogHandlers.updateView(viewCatalog, ident, request)); + } + break; + } + + case RENAME_VIEW: + { + if (null != viewCatalog) { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameView(viewCatalog, request); + return null; + } + break; + } + + case DROP_VIEW: + { + if (null != viewCatalog) { + CatalogHandlers.dropView(viewCatalog, viewIdentFromPathVars(vars)); + return null; + } + break; + } + default: } @@ -568,4 +649,9 @@ private static TableIdentifier identFromPathVars(Map pathVars) { return TableIdentifier.of( namespaceFromPathVars(pathVars), RESTUtil.decodeString(pathVars.get("table"))); } + + private static TableIdentifier viewIdentFromPathVars(Map pathVars) { + return TableIdentifier.of( + namespaceFromPathVars(pathVars), RESTUtil.decodeString(pathVars.get("view"))); + } } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java new file mode 100644 index 000000000000..2d4545c6172d --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.view.ViewCatalogTests; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; + +public class TestRESTViewCatalog extends ViewCatalogTests { + private static final ObjectMapper MAPPER = RESTObjectMapper.mapper(); + + @TempDir private Path temp; + + private RESTCatalog restCatalog; + private InMemoryCatalog backendCatalog; + private Server httpServer; + + @BeforeEach + public void createCatalog() throws Exception { + File warehouse = temp.toFile(); + + this.backendCatalog = new InMemoryCatalog(); + this.backendCatalog.initialize( + "in-memory", + ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath())); + + RESTCatalogAdapter adaptor = + new RESTCatalogAdapter(backendCatalog) { + @Override + public T execute( + HTTPMethod method, + String path, + Map queryParams, + Object body, + Class responseType, + Map headers, + Consumer errorHandler) { + Object request = roundTripSerialize(body, "request"); + T response = + super.execute( + method, path, queryParams, request, responseType, headers, errorHandler); + T responseAfterSerialization = roundTripSerialize(response, "response"); + return responseAfterSerialization; + } + }; + + RESTCatalogServlet servlet = new RESTCatalogServlet(adaptor); + ServletContextHandler servletContext = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + servletContext.setContextPath("/"); + ServletHolder servletHolder = new ServletHolder(servlet); + servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic"); + servletContext.addServlet(servletHolder, "/*"); + servletContext.setVirtualHosts(null); + servletContext.setGzipHandler(new GzipHandler()); + + this.httpServer = new Server(0); + httpServer.setHandler(servletContext); + httpServer.start(); + + SessionCatalog.SessionContext context = + new SessionCatalog.SessionContext( + UUID.randomUUID().toString(), + "user", + ImmutableMap.of("credential", "user:12345"), + ImmutableMap.of()); + + this.restCatalog = + new RESTCatalog( + context, + (config) -> HTTPClient.builder(config).uri(config.get(CatalogProperties.URI)).build()); + restCatalog.initialize( + "prod", + ImmutableMap.of( + CatalogProperties.URI, httpServer.getURI().toString(), "credential", "catalog:12345")); + } + + @SuppressWarnings("unchecked") + public static T roundTripSerialize(T payload, String description) { + if (payload != null) { + try { + if (payload instanceof RESTMessage) { + return (T) MAPPER.readValue(MAPPER.writeValueAsString(payload), payload.getClass()); + } else { + // use Map so that Jackson doesn't try to instantiate ImmutableMap from payload.getClass() + return (T) MAPPER.readValue(MAPPER.writeValueAsString(payload), Map.class); + } + } catch (JsonProcessingException e) { + throw new RuntimeException( + String.format("Failed to serialize and deserialize %s: %s", description, payload), e); + } + } + return null; + } + + @AfterEach + public void closeCatalog() throws Exception { + if (restCatalog != null) { + restCatalog.close(); + } + + if (backendCatalog != null) { + backendCatalog.close(); + } + + if (httpServer != null) { + httpServer.stop(); + httpServer.join(); + } + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected Catalog tableCatalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java index e0e61a594e7d..e468c28c2136 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java @@ -143,4 +143,51 @@ public void testRegister() { .isEqualTo("v1/ws/catalog/namespaces/ns/register"); Assertions.assertThat(withoutPrefix.register(ns)).isEqualTo("v1/namespaces/ns/register"); } + + @Test + public void testViews() { + Namespace ns = Namespace.of("ns"); + Assertions.assertThat(withPrefix.views(ns)).isEqualTo("v1/ws/catalog/namespaces/ns/views"); + Assertions.assertThat(withoutPrefix.views(ns)).isEqualTo("v1/namespaces/ns/views"); + } + + @Test + public void testViewsWithSlash() { + Namespace ns = Namespace.of("n/s"); + Assertions.assertThat(withPrefix.views(ns)).isEqualTo("v1/ws/catalog/namespaces/n%2Fs/views"); + Assertions.assertThat(withoutPrefix.views(ns)).isEqualTo("v1/namespaces/n%2Fs/views"); + } + + @Test + public void testViewsWithMultipartNamespace() { + Namespace ns = Namespace.of("n", "s"); + Assertions.assertThat(withPrefix.views(ns)).isEqualTo("v1/ws/catalog/namespaces/n%1Fs/views"); + Assertions.assertThat(withoutPrefix.views(ns)).isEqualTo("v1/namespaces/n%1Fs/views"); + } + + @Test + public void testView() { + TableIdentifier ident = TableIdentifier.of("ns", "view-name"); + Assertions.assertThat(withPrefix.view(ident)) + .isEqualTo("v1/ws/catalog/namespaces/ns/views/view-name"); + Assertions.assertThat(withoutPrefix.view(ident)).isEqualTo("v1/namespaces/ns/views/view-name"); + } + + @Test + public void testViewWithSlash() { + TableIdentifier ident = TableIdentifier.of("n/s", "vi/ew-name"); + Assertions.assertThat(withPrefix.view(ident)) + .isEqualTo("v1/ws/catalog/namespaces/n%2Fs/views/vi%2Few-name"); + Assertions.assertThat(withoutPrefix.view(ident)) + .isEqualTo("v1/namespaces/n%2Fs/views/vi%2Few-name"); + } + + @Test + public void testViewWithMultipartNamespace() { + TableIdentifier ident = TableIdentifier.of("n", "s", "view-name"); + Assertions.assertThat(withPrefix.view(ident)) + .isEqualTo("v1/ws/catalog/namespaces/n%1Fs/views/view-name"); + Assertions.assertThat(withoutPrefix.view(ident)) + .isEqualTo("v1/namespaces/n%1Fs/views/view-name"); + } } diff --git a/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateViewRequestParser.java b/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateViewRequestParser.java new file mode 100644 index 000000000000..8edea2467c02 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateViewRequestParser.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.requests; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.ViewMetadata; +import org.junit.jupiter.api.Test; + +public class TestCreateViewRequestParser { + + @Test + public void nullAndEmptyCheck() { + assertThatThrownBy(() -> CreateViewRequestParser.toJson(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid create view request: null"); + + assertThatThrownBy(() -> CreateViewRequestParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse create view request from null object"); + } + + @Test + public void missingFields() { + assertThatThrownBy(() -> CreateViewRequestParser.fromJson("{}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing string: name"); + + assertThatThrownBy(() -> CreateViewRequestParser.fromJson("{\"name\": \"view-name\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing field: metadata"); + } + + @Test + public void roundTripSerde() { + ViewMetadata viewMetadata = + ViewMetadata.builder() + .setLocation("location") + .addSchema(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "create") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(2) + .timestampMillis(24L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(3) + .build(); + + CreateViewRequest request = + ImmutableCreateViewRequest.builder().name("view-name").metadata(viewMetadata).build(); + String expectedJson = + "{\n" + + " \"name\" : \"view-name\",\n" + + " \"metadata\" : {\n" + + " \"format-version\" : 1,\n" + + " \"location\" : \"location\",\n" + + " \"properties\" : { },\n" + + " \"schemas\" : [ {\n" + + " \"type\" : \"struct\",\n" + + " \"schema-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"id\" : 1,\n" + + " \"name\" : \"x\",\n" + + " \"required\" : true,\n" + + " \"type\" : \"long\"\n" + + " } ]\n" + + " } ],\n" + + " \"current-version-id\" : 3,\n" + + " \"versions\" : [ {\n" + + " \"version-id\" : 1,\n" + + " \"timestamp-ms\" : 23,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"create\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 2,\n" + + " \"timestamp-ms\" : 24,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 3,\n" + + " \"timestamp-ms\" : 25,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " } ],\n" + + " \"version-log\" : [ {\n" + + " \"timestamp-ms\" : 23,\n" + + " \"version-id\" : 1\n" + + " }, {\n" + + " \"timestamp-ms\" : 24,\n" + + " \"version-id\" : 2\n" + + " }, {\n" + + " \"timestamp-ms\" : 25,\n" + + " \"version-id\" : 3\n" + + " } ]\n" + + " }\n" + + "}"; + + String json = CreateViewRequestParser.toJson(request, true); + assertThat(json).isEqualTo(expectedJson); + + // can't do an equality comparison because Schema doesn't implement equals/hashCode + assertThat(CreateViewRequestParser.toJson(CreateViewRequestParser.fromJson(json), true)) + .isEqualTo(expectedJson); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java new file mode 100644 index 000000000000..77c0d5e2125f --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.iceberg.view.ViewMetadata; +import org.junit.jupiter.api.Test; + +public class TestLoadViewResponseParser { + + @Test + public void nullAndEmptyCheck() { + assertThatThrownBy(() -> LoadViewResponseParser.toJson(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid load view response: null"); + + assertThatThrownBy(() -> LoadViewResponseParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse load view response from null object"); + } + + @Test + public void missingFields() { + assertThatThrownBy(() -> LoadViewResponseParser.fromJson("{}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing string: metadata-location"); + + assertThatThrownBy( + () -> LoadViewResponseParser.fromJson("{\"metadata-location\": \"custom-location\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing field: metadata"); + + assertThatThrownBy( + () -> LoadViewResponseParser.fromJson("{\"metadata-location\": \"custom-location\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing field: metadata"); + } + + @Test + public void roundTripSerde() { + ViewMetadata viewMetadata = + ViewMetadata.builder() + .setLocation("location") + .addSchema(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "create") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(2) + .timestampMillis(24L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(3) + .build(); + + LoadViewResponse response = + ImmutableLoadViewResponse.builder() + .metadata(viewMetadata) + .metadataLocation("custom-location") + .build(); + String expectedJson = + "{\n" + + " \"metadata-location\" : \"custom-location\",\n" + + " \"metadata\" : {\n" + + " \"format-version\" : 1,\n" + + " \"location\" : \"location\",\n" + + " \"properties\" : { },\n" + + " \"schemas\" : [ {\n" + + " \"type\" : \"struct\",\n" + + " \"schema-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"id\" : 1,\n" + + " \"name\" : \"x\",\n" + + " \"required\" : true,\n" + + " \"type\" : \"long\"\n" + + " } ]\n" + + " } ],\n" + + " \"current-version-id\" : 3,\n" + + " \"versions\" : [ {\n" + + " \"version-id\" : 1,\n" + + " \"timestamp-ms\" : 23,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"create\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 2,\n" + + " \"timestamp-ms\" : 24,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 3,\n" + + " \"timestamp-ms\" : 25,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " } ],\n" + + " \"version-log\" : [ {\n" + + " \"timestamp-ms\" : 23,\n" + + " \"version-id\" : 1\n" + + " }, {\n" + + " \"timestamp-ms\" : 24,\n" + + " \"version-id\" : 2\n" + + " }, {\n" + + " \"timestamp-ms\" : 25,\n" + + " \"version-id\" : 3\n" + + " } ]\n" + + " }\n" + + "}"; + + String json = LoadViewResponseParser.toJson(response, true); + assertThat(json).isEqualTo(expectedJson); + // can't do an equality comparison because Schema doesn't implement equals/hashCode + assertThat(LoadViewResponseParser.toJson(LoadViewResponseParser.fromJson(json), true)) + .isEqualTo(expectedJson); + } + + @Test + public void roundTripSerdeWithConfig() { + ViewMetadata viewMetadata = + ViewMetadata.builder() + .setLocation("location") + .addSchema(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "create") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(2) + .timestampMillis(24L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(3) + .timestampMillis(25L) + .putSummary("operation", "replace") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(3) + .build(); + + LoadViewResponse response = + ImmutableLoadViewResponse.builder() + .metadata(viewMetadata) + .metadataLocation("custom-location") + .config(ImmutableMap.of("key1", "val1", "key2", "val2")) + .build(); + String expectedJson = + "{\n" + + " \"metadata-location\" : \"custom-location\",\n" + + " \"metadata\" : {\n" + + " \"format-version\" : 1,\n" + + " \"location\" : \"location\",\n" + + " \"properties\" : { },\n" + + " \"schemas\" : [ {\n" + + " \"type\" : \"struct\",\n" + + " \"schema-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"id\" : 1,\n" + + " \"name\" : \"x\",\n" + + " \"required\" : true,\n" + + " \"type\" : \"long\"\n" + + " } ]\n" + + " } ],\n" + + " \"current-version-id\" : 3,\n" + + " \"versions\" : [ {\n" + + " \"version-id\" : 1,\n" + + " \"timestamp-ms\" : 23,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"create\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 2,\n" + + " \"timestamp-ms\" : 24,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " }, {\n" + + " \"version-id\" : 3,\n" + + " \"timestamp-ms\" : 25,\n" + + " \"schema-id\" : 1,\n" + + " \"summary\" : {\n" + + " \"operation\" : \"replace\"\n" + + " },\n" + + " \"default-namespace\" : [ \"ns\" ],\n" + + " \"representations\" : [ ]\n" + + " } ],\n" + + " \"version-log\" : [ {\n" + + " \"timestamp-ms\" : 23,\n" + + " \"version-id\" : 1\n" + + " }, {\n" + + " \"timestamp-ms\" : 24,\n" + + " \"version-id\" : 2\n" + + " }, {\n" + + " \"timestamp-ms\" : 25,\n" + + " \"version-id\" : 3\n" + + " } ]\n" + + " },\n" + + " \"config\" : {\n" + + " \"key1\" : \"val1\",\n" + + " \"key2\" : \"val2\"\n" + + " }\n" + + "}"; + + String json = LoadViewResponseParser.toJson(response, true); + assertThat(json).isEqualTo(expectedJson); + // can't do an equality comparison because Schema doesn't implement equals/hashCode + assertThat(LoadViewResponseParser.toJson(LoadViewResponseParser.fromJson(json), true)) + .isEqualTo(expectedJson); + } +} diff --git a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java index 6e7efd2c70ef..403e088088bc 100644 --- a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java @@ -197,6 +197,7 @@ public void createViewThatAlreadyExists() { .buildView(viewIdentifier) .withSchema(OTHER_SCHEMA) .withQuery("spark", "select * from ns.tbl") + .withDefaultNamespace(viewIdentifier.namespace()) .create()) .isInstanceOf(AlreadyExistsException.class) .hasMessageStartingWith("View already exists: ns.view"); @@ -230,6 +231,7 @@ public void createViewThatAlreadyExistsAsTable() { catalog() .buildView(viewIdentifier) .withSchema(OTHER_SCHEMA) + .withDefaultNamespace(viewIdentifier.namespace()) .withQuery("spark", "select * from ns.tbl") .create()) .isInstanceOf(AlreadyExistsException.class) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5fc9fe6e688b..c08a7f2dfd39 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -209,6 +209,31 @@ class MetadataLog(BaseModel): __root__: List[MetadataLogItem] +class SQLViewRepresentation(BaseModel): + type: str + sql: str + dialect: str + + +class ViewRepresentation(BaseModel): + __root__: SQLViewRepresentation + + +class ViewHistoryEntry(BaseModel): + version_id: int = Field(..., alias='version-id') + timestamp_ms: int = Field(..., alias='timestamp-ms') + + +class ViewVersion(BaseModel): + version_id: int = Field(..., alias='version-id') + timestamp_ms: int = Field(..., alias='timestamp-ms') + schema_id: int = Field(..., alias='schema-id') + summary: Dict[str, str] + representations: List[ViewRepresentation] + default_catalog: Optional[str] = Field(None, alias='default-catalog') + default_namespace: Namespace = Field(..., alias='default-namespace') + + class BaseUpdate(BaseModel): action: Literal[ 'upgrade-format-version', @@ -225,6 +250,8 @@ class BaseUpdate(BaseModel): 'set-location', 'set-properties', 'remove-properties', + 'add-view-version', + 'set-current-view-version', ] @@ -292,6 +319,14 @@ class RemovePropertiesUpdate(BaseUpdate): removals: List[str] +class AddViewVersionUpdate(BaseUpdate): + view_version: ViewVersion = Field(..., alias='view-version') + + +class SetCurrentViewVersionUpdate(BaseUpdate): + view_version_id: int = Field(..., alias='view-version-id') + + class TableRequirement(BaseModel): """ Assertions from the client that must be valid for the commit to succeed. Assertions are identified by `type` - @@ -621,6 +656,16 @@ class TableMetadata(BaseModel): metadata_log: Optional[MetadataLog] = Field(None, alias='metadata-log') +class ViewMetadata(BaseModel): + format_version: int = Field(..., alias='format-version', ge=1, le=1) + location: str + current_version_id: int = Field(..., alias='current-version-id') + versions: List[ViewVersion] + version_log: List[ViewHistoryEntry] = Field(..., alias='version-log') + schemas: List[Schema] + properties: Dict[str, str] + + class AddSchemaUpdate(BaseUpdate): schema_: Schema = Field(..., alias='schema') last_column_id: Optional[int] = Field( @@ -649,6 +694,18 @@ class TableUpdate(BaseModel): ] +class ViewUpdate(BaseModel): + __root__: Union[ + UpgradeFormatVersionUpdate, + AddSchemaUpdate, + SetLocationUpdate, + SetPropertiesUpdate, + RemovePropertiesUpdate, + AddViewVersionUpdate, + SetCurrentViewVersionUpdate, + ] + + class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -696,6 +753,13 @@ class CommitTableRequest(BaseModel): updates: List[TableUpdate] +class CommitViewRequest(BaseModel): + identifier: Optional[TableIdentifier] = Field( + None, description='View identifier to update' + ) + updates: List[ViewUpdate] + + class CommitTransactionRequest(BaseModel): table_changes: List[CommitTableRequest] = Field(..., alias='table-changes') @@ -710,6 +774,46 @@ class CreateTableRequest(BaseModel): properties: Optional[Dict[str, str]] = None +class CreateViewRequest(BaseModel): + name: str + metadata: ViewMetadata + + +class LoadViewResult(BaseModel): + """ + Result used when a view is successfully loaded. + + + The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. + Clients can check whether metadata has changed by comparing metadata locations after the view has been created. + + + The `config` map returns view-specific configuration for the view's resources, including its HTTP client and FileIO. + For example, config may contain a specific FileIO implementation class for the view depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + + ## AWS Configurations + + The following configurations should be respected when working with views stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + + """ + + metadata_location: str = Field(..., alias='metadata-location') + metadata: ViewMetadata + config: Optional[Dict[str, str]] = None + + class ReportMetricsRequest2(BaseModel): __root__: Union[ReportMetricsRequest, ReportMetricsRequest1] @@ -746,6 +850,7 @@ class ReportMetricsRequest(ScanReport): MapType.update_forward_refs() Expression.update_forward_refs() TableMetadata.update_forward_refs() +ViewMetadata.update_forward_refs() AddSchemaUpdate.update_forward_refs() CreateTableRequest.update_forward_refs() ReportMetricsRequest2.update_forward_refs() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c371afc5583a..0b394532fc50 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1014,6 +1014,357 @@ paths: } } + /v1/{prefix}/namespaces/{namespace}/views: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + + get: + tags: + - Catalog API + summary: List all view identifiers underneath a given namespace + description: Return all view identifiers under this namespace + operationId: listViews + responses: + 200: + $ref: '#/components/responses/ListTablesResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Create a view in the given namespace + description: + Create a view in the given namespace. + operationId: createView + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateViewRequest' + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The view already exists + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + NamespaceAlreadyExists: + $ref: '#/components/examples/ViewAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/namespaces/{namespace}/views/{view}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/view' + + get: + tags: + - Catalog API + summary: Load a view from the catalog + operationId: loadView + description: + Load a view from the catalog. + + + The response contains both configuration and table metadata. The configuration, if non-empty is used + as additional configuration for the view that overrides catalog configuration. For example, this + configuration may change the FileIO implementation to be used for the view. + + + The response also contains the view's full metadata, matching the view metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + view. The configuration key "token" is used to pass an access token to be used as a bearer token + for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + post: + tags: + - Catalog API + summary: Replace a view + operationId: replaceView + description: + Commit updates to a view. + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CommitViewRequest' + responses: + 200: + $ref: '#/components/responses/LoadViewResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToUpdateDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 409: + description: + Conflict - CommitFailedException. The client may retry. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 500: + description: + An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500 + } + } + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 502: + description: + A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502 + } + } + 504: + description: + A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504 + } + } + 5XX: + description: + A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: { + "error": { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502 + } + } + + delete: + tags: + - Catalog API + summary: Drop a view from the catalog + operationId: dropView + description: Remove a view from the catalog + responses: + 204: + description: Success, no content + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchViewException, view to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + head: + tags: + - Catalog API + summary: Check if a view exists + operationId: viewExists + description: + Check if a view exists within a given namespace. This request does not return a response body. + responses: + 200: + description: OK - View Exists + 400: + description: Bad Request + 401: + description: Unauthorized + 404: + description: Not Found + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + + /v1/{prefix}/views/rename: + parameters: + - $ref: '#/components/parameters/prefix' + + post: + tags: + - Catalog API + summary: Rename a view from its current name to a new name + description: + Rename a view from one identifier to another. It's valid to move a view + across namespaces, but the server implementation is not required to support it. + operationId: renameView + requestBody: + description: Current view identifier to rename and new view identifier to rename to + content: + application/json: + schema: + $ref: '#/components/schemas/RenameTableRequest' + examples: + RenameViewSameNamespace: + $ref: '#/components/examples/RenameViewSameNamespace' + required: true + responses: + 200: + description: OK + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 404: + description: + Not Found + - NoSuchViewException, view to rename does not exist + - NoSuchNamespaceException, The target namespace of the new table identifier does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + examples: + ViewToRenameDoesNotExist: + $ref: '#/components/examples/NoSuchViewError' + NamespaceToRenameToDoesNotExist: + $ref: '#/components/examples/NoSuchNamespaceError' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 409: + description: Conflict - The target view identifier to rename to already exists + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorModel' + example: + $ref: '#/components/examples/ViewAlreadyExistsError' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + components: ####################################################### @@ -1052,6 +1403,15 @@ components: type: string example: "sales" + view: + name: view + in: path + description: A view name + required: true + schema: + type: string + example: "sales-view" + ############################## # Application Schema Objects # ############################## @@ -1630,6 +1990,102 @@ components: metadata-log: $ref: '#/components/schemas/MetadataLog' + SQLViewRepresentation: + type: object + required: + - type + - sql + - dialect + properties: + type: + type: string + sql: + type: string + dialect: + type: string + + ViewRepresentation: + oneOf: + - $ref: '#/components/schemas/SQLViewRepresentation' + + ViewHistoryEntry: + type: object + required: + - version-id + - timestamp-ms + properties: + version-id: + type: integer + timestamp-ms: + type: integer + format: int64 + + ViewVersion: + type: object + required: + - version-id + - timestamp-ms + - schema-id + - summary + - representations + - default-namespace + properties: + version-id: + type: integer + timestamp-ms: + type: integer + format: int64 + schema-id: + type: integer + summary: + type: object + additionalProperties: + type: string + representations: + type: array + items: + $ref: '#/components/schemas/ViewRepresentation' + default-catalog: + type: string + default-namespace: + $ref: '#/components/schemas/Namespace' + + ViewMetadata: + type: object + required: + - format-version + - location + - current-version-id + - versions + - version-log + - schemas + - properties + properties: + format-version: + type: integer + minimum: 1 + maximum: 1 + location: + type: string + current-version-id: + type: integer + versions: + type: array + items: + $ref: '#/components/schemas/ViewVersion' + version-log: + type: array + items: + $ref: '#/components/schemas/ViewHistoryEntry' + schemas: + type: array + items: + $ref: '#/components/schemas/Schema' + properties: + type: object + additionalProperties: + type: string + BaseUpdate: type: object required: @@ -1652,6 +2108,8 @@ components: - set-location - set-properties - remove-properties + - add-view-version + - set-current-view-version UpgradeFormatVersionUpdate: allOf: @@ -1807,6 +2265,26 @@ components: items: type: string + AddViewVersionUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + - type: object + required: + - view-version + properties: + view-version: + $ref: '#/components/schemas/ViewVersion' + + SetCurrentViewVersionUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + - type: object + required: + - view-version-id + properties: + view-version-id: + type: integer + TableUpdate: anyOf: - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' @@ -1824,6 +2302,16 @@ components: - $ref: '#/components/schemas/SetPropertiesUpdate' - $ref: '#/components/schemas/RemovePropertiesUpdate' + ViewUpdate: + anyOf: + - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' + - $ref: '#/components/schemas/AddSchemaUpdate' + - $ref: '#/components/schemas/SetLocationUpdate' + - $ref: '#/components/schemas/SetPropertiesUpdate' + - $ref: '#/components/schemas/RemovePropertiesUpdate' + - $ref: '#/components/schemas/AddViewVersionUpdate' + - $ref: '#/components/schemas/SetCurrentViewVersionUpdate' + TableRequirement: description: Assertions from the client that must be valid for the commit to succeed. Assertions are identified by `type` - @@ -1934,6 +2422,19 @@ components: items: $ref: '#/components/schemas/TableUpdate' + CommitViewRequest: + type: object + required: + - updates + properties: + identifier: + description: View identifier to update + $ref: '#/components/schemas/TableIdentifier' + updates: + type: array + items: + $ref: '#/components/schemas/ViewUpdate' + CommitTransactionRequest: type: object required: @@ -1979,6 +2480,58 @@ components: metadata-location: type: string + CreateViewRequest: + type: object + required: + - name + - metadata + properties: + name: + type: string + metadata: + $ref: '#/components/schemas/ViewMetadata' + + LoadViewResult: + description: | + Result used when a view is successfully loaded. + + + The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. + Clients can check whether metadata has changed by comparing metadata locations after the view has been created. + + + The `config` map returns view-specific configuration for the view's resources, including its HTTP client and FileIO. + For example, config may contain a specific FileIO implementation class for the view depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + + ## AWS Configurations + + The following configurations should be respected when working with views stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + metadata: + $ref: '#/components/schemas/ViewMetadata' + config: + type: object + additionalProperties: + type: string + TokenType: type: string enum: @@ -2589,6 +3142,13 @@ components: schema: $ref: '#/components/schemas/LoadTableResult' + LoadViewResponse: + description: View metadata result when loading a view + content: + application/json: + schema: + $ref: '#/components/schemas/LoadViewResult' + CommitTableResponse: description: Response used when a table is successfully updated. @@ -2653,7 +3213,7 @@ components: } NoSuchTableError: - summary: The requested table does not + summary: The requested table does not exist value: { "error": { "message": "The given table does not exist", @@ -2662,6 +3222,16 @@ components: } } + NoSuchViewError: + summary: The requested view does not exist + value: { + "error": { + "message": "The given view does not exist", + "type": "NoSuchViewException", + "code": 404 + } + } + NoSuchNamespaceError: summary: The requested namespace does not exist value: { @@ -2679,6 +3249,13 @@ components: "destination": { "namespace": ["accounting", "tax"], "name": "owed" } } + RenameViewSameNamespace: + summary: Rename a view in the same namespace + value: { + "source": { "namespace": [ "accounting", "tax" ], "name": "paid-view" }, + "destination": { "namespace": [ "accounting", "tax" ], "name": "owed-view" } + } + TableAlreadyExistsError: summary: The requested table identifier already exists value: { @@ -2689,6 +3266,16 @@ components: } } + ViewAlreadyExistsError: + summary: The requested view identifier already exists + value: { + "error": { + "message": "The given view already exists", + "type": "AlreadyExistsException", + "code": 409 + } + } + # This is an example response and is not meant to be prescriptive regarding the message or type. UnprocessableEntityDuplicateKey: summary: