From db8a14abef75fdc73d303b7bd417e6e69a355cba Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 13 Jan 2025 05:52:30 +0800 Subject: [PATCH 01/17] [#6177] improve(CLI): Refactor ownership commands in Gravitino CLI (#6188) ### What changes were proposed in this pull request? Refactor ownership commands in Gravitino CLI. ### Why are the changes needed? Fix: #6177 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? local test. --- .../gravitino/cli/GravitinoCommandLine.java | 41 +----- .../gravitino/cli/OwnerCommandHandler.java | 128 ++++++++++++++++++ .../gravitino/cli/TestOwnerCommands.java | 79 +++++++++++ 3 files changed, 208 insertions(+), 40 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 675a96d36a8..9c9ce6810ba 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -127,7 +127,7 @@ private void executeCommand() { if (CommandActions.HELP.equals(command)) { handleHelpCommand(); } else if (line.hasOption(GravitinoOptions.OWNER)) { - handleOwnerCommand(); + new OwnerCommandHandler(this, line, command, ignore, entity).handle(); } else if (entity.equals(CommandEntities.COLUMN)) { handleColumnCommand(); } else if (entity.equals(CommandEntities.TABLE)) { @@ -554,45 +554,6 @@ private void handleHelpCommand() { } } - /** - * Handles the command execution for Objects based on command type and the command line options. - */ - private void handleOwnerCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String entityName = line.getOptionValue(GravitinoOptions.NAME); - - Command.setAuthenticationMode(auth, userName); - - switch (command) { - case CommandActions.DETAILS: - newOwnerDetails(url, ignore, metalake, entityName, entity).handle(); - break; - - case CommandActions.SET: - { - String owner = line.getOptionValue(GravitinoOptions.USER); - String group = line.getOptionValue(GravitinoOptions.GROUP); - - if (owner != null && group == null) { - newSetOwner(url, ignore, metalake, entityName, entity, owner, false).handle(); - } else if (owner == null && group != null) { - newSetOwner(url, ignore, metalake, entityName, entity, group, true).handle(); - } else { - System.err.println(ErrorMessages.INVALID_SET_COMMAND); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - break; - } - } - /** * Handles the command execution for filesets based on command type and the command line options. */ diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java new file mode 100644 index 00000000000..7e41fb478ae --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Owner based on command type and the command line options. */ +public class OwnerCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String entityName; + private final String owner; + private final String group; + private final String entity; + + /** + * Constructs a {@link OwnerCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + * @param entity The entity to execute the command on. + */ + public OwnerCommandHandler( + GravitinoCommandLine gravitinoCommandLine, + CommandLine line, + String command, + boolean ignore, + String entity) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.owner = line.getOptionValue(GravitinoOptions.USER); + this.group = line.getOptionValue(GravitinoOptions.GROUP); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.entityName = name.getName(); + this.entity = entity; + } + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + if (entityName == null && !CommandEntities.METALAKE.equals(entity)) { + System.err.println(ErrorMessages.MISSING_NAME); + Main.exit(-1); + } + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine + .newOwnerDetails(url, ignore, metalake, entityName, entity) + .validate() + .handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + if (owner != null && group == null) { + gravitinoCommandLine + .newSetOwner(url, ignore, metalake, entityName, entity, owner, false) + .validate() + .handle(); + } else if (owner == null && group != null) { + gravitinoCommandLine + .newSetOwner(url, ignore, metalake, entityName, entity, group, true) + .validate() + .handle(); + } else { + System.err.println(ErrorMessages.INVALID_SET_COMMAND); + Main.exit(-1); + } + } +} diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java index 0c2b2cf91e5..12f617380ca 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java @@ -19,16 +19,25 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.OwnerDetails; import org.apache.gravitino.cli.commands.SetOwner; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,10 +45,23 @@ class TestOwnerCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -67,6 +89,7 @@ void testSetOwnerUserCommand() { "catalog", "admin", false); + doReturn(mockSetOwner).when(mockSetOwner).validate(); commandLine.handleCommandLine(); verify(mockSetOwner).handle(); } @@ -96,6 +119,7 @@ void testSetOwnerGroupCommand() { "catalog", "ITdept", true); + doReturn(mockSetOwner).when(mockSetOwner).validate(); commandLine.handleCommandLine(); verify(mockSetOwner).handle(); } @@ -116,7 +140,62 @@ void testOwnerDetailsCommand() { .when(commandLine) .newOwnerDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "postgres", "catalog"); + doReturn(mockOwnerDetails).when(mockOwnerDetails).validate(); commandLine.handleCommandLine(); verify(mockOwnerDetails).handle(); } + + @Test + void testOwnerDetailsCommandWithoutName() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.OWNER)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newOwnerDetails( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq(null), + eq(CommandEntities.CATALOG)); + + String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(ErrorMessages.MISSING_NAME, errOutput); + } + + @Test + void testSetOwnerUserCommandWithoutUserAndGroup() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("postgres"); + when(mockCommandLine.hasOption(GravitinoOptions.USER)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.GROUP)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.OWNER)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.SET)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newSetOwner( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("postgres"), + eq(CommandEntities.CATALOG), + isNull(), + eq(false)); + String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(ErrorMessages.INVALID_SET_COMMAND, errOutput); + } } From 80d6daa3f89464319942052f4bffd9e931095184 Mon Sep 17 00:00:00 2001 From: TungYuChiang <75083792+TungYuChiang@users.noreply.github.com> Date: Mon, 13 Jan 2025 05:54:16 +0800 Subject: [PATCH 02/17] [#6149] improve(CLI): Refactor column commands in Gavitino CLI (#6190) ### What changes were proposed in this pull request? Refactor column commands CLI ### Why are the changes needed? Fix: #6149 ### Does this PR introduce _any_ user-facing change? None ### How was this patch tested? Tested locally --- .../gravitino/cli/ColumnCommandHandler.java | 236 ++++++++++++++++++ .../gravitino/cli/GravitinoCommandLine.java | 137 +--------- 2 files changed, 237 insertions(+), 136 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java new file mode 100644 index 00000000000..96f056c1a3c --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Columns based on command type and the command line options. */ +public class ColumnCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String catalog; + private final String schema; + private final String table; + private String column; + + /** + * Constructs a {@link ColumnCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public ColumnCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = gravitinoCommandLine.getUrl(); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.catalog = name.getCatalogName(); + this.schema = name.getSchemaName(); + this.table = name.getTableName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(gravitinoCommandLine.getAuth(), userName); + + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (table == null) missingEntities.add(CommandEntities.TABLE); + + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); + handleListCommand(); + return; + } + + this.column = name.getColumnName(); + if (column == null) missingEntities.add(CommandEntities.COLUMN); + checkEntities(missingEntities); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + gravitinoCommandLine + .newColumnAudit(url, ignore, metalake, catalog, schema, table, column) + .validate() + .handle(); + } else { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + String position = line.getOptionValue(GravitinoOptions.POSITION); + boolean nullable = + !line.hasOption(GravitinoOptions.NULL) + || line.getOptionValue(GravitinoOptions.NULL).equals("true"); + boolean autoIncrement = + line.hasOption(GravitinoOptions.AUTO) + && line.getOptionValue(GravitinoOptions.AUTO).equals("true"); + String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); + + gravitinoCommandLine + .newAddColumn( + url, + ignore, + metalake, + catalog, + schema, + table, + column, + datatype, + comment, + position, + nullable, + autoIncrement, + defaultValue) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + gravitinoCommandLine + .newDeleteColumn(url, ignore, metalake, catalog, schema, table, column) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateColumnComment(url, ignore, metalake, catalog, schema, table, column, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateColumnName(url, ignore, metalake, catalog, schema, table, column, newName) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DATATYPE) && !line.hasOption(GravitinoOptions.DEFAULT)) { + String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); + gravitinoCommandLine + .newUpdateColumnDatatype(url, ignore, metalake, catalog, schema, table, column, datatype) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.POSITION)) { + String position = line.getOptionValue(GravitinoOptions.POSITION); + gravitinoCommandLine + .newUpdateColumnPosition(url, ignore, metalake, catalog, schema, table, column, position) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.NULL)) { + boolean nullable = line.getOptionValue(GravitinoOptions.NULL).equals("true"); + gravitinoCommandLine + .newUpdateColumnNullability( + url, ignore, metalake, catalog, schema, table, column, nullable) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.AUTO)) { + boolean autoIncrement = line.getOptionValue(GravitinoOptions.AUTO).equals("true"); + gravitinoCommandLine + .newUpdateColumnAutoIncrement( + url, ignore, metalake, catalog, schema, table, column, autoIncrement) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DEFAULT)) { + String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); + String dataType = line.getOptionValue(GravitinoOptions.DATATYPE); + gravitinoCommandLine + .newUpdateColumnDefault( + url, ignore, metalake, catalog, schema, table, column, defaultValue, dataType) + .validate() + .handle(); + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + gravitinoCommandLine + .newListColumns(url, ignore, metalake, catalog, schema, table) + .validate() + .handle(); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 9c9ce6810ba..b883502e805 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -129,7 +129,7 @@ private void executeCommand() { } else if (line.hasOption(GravitinoOptions.OWNER)) { new OwnerCommandHandler(this, line, command, ignore, entity).handle(); } else if (entity.equals(CommandEntities.COLUMN)) { - handleColumnCommand(); + new ColumnCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TABLE)) { new TableCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.SCHEMA)) { @@ -401,141 +401,6 @@ private String getOneTag(String[] tags) { return tags[0]; } - /** - * Handles the command execution for Columns based on command type and the command line options. - */ - private void handleColumnCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - String table = name.getTableName(); - - Command.setAuthenticationMode(auth, userName); - - List missingEntities = Lists.newArrayList(); - if (catalog == null) missingEntities.add(CommandEntities.CATALOG); - if (schema == null) missingEntities.add(CommandEntities.SCHEMA); - if (table == null) missingEntities.add(CommandEntities.TABLE); - - if (CommandActions.LIST.equals(command)) { - checkEntities(missingEntities); - newListColumns(url, ignore, metalake, catalog, schema, table).validate().handle(); - return; - } - - String column = name.getColumnName(); - if (column == null) missingEntities.add(CommandEntities.COLUMN); - checkEntities(missingEntities); - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newColumnAudit(url, ignore, metalake, catalog, schema, table, column).validate().handle(); - } else { - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - } - break; - - case CommandActions.CREATE: - { - String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - String position = line.getOptionValue(GravitinoOptions.POSITION); - boolean nullable = - !line.hasOption(GravitinoOptions.NULL) - || line.getOptionValue(GravitinoOptions.NULL).equals("true"); - boolean autoIncrement = - line.hasOption(GravitinoOptions.AUTO) - && line.getOptionValue(GravitinoOptions.AUTO).equals("true"); - String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); - - newAddColumn( - url, - ignore, - metalake, - catalog, - schema, - table, - column, - datatype, - comment, - position, - nullable, - autoIncrement, - defaultValue) - .validate() - .handle(); - break; - } - - case CommandActions.DELETE: - newDeleteColumn(url, ignore, metalake, catalog, schema, table, column).validate().handle(); - break; - - case CommandActions.UPDATE: - { - if (line.hasOption(GravitinoOptions.COMMENT)) { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateColumnComment(url, ignore, metalake, catalog, schema, table, column, comment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateColumnName(url, ignore, metalake, catalog, schema, table, column, newName) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.DATATYPE) - && !line.hasOption(GravitinoOptions.DEFAULT)) { - String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); - newUpdateColumnDatatype(url, ignore, metalake, catalog, schema, table, column, datatype) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.POSITION)) { - String position = line.getOptionValue(GravitinoOptions.POSITION); - newUpdateColumnPosition(url, ignore, metalake, catalog, schema, table, column, position) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.NULL)) { - boolean nullable = line.getOptionValue(GravitinoOptions.NULL).equals("true"); - newUpdateColumnNullability( - url, ignore, metalake, catalog, schema, table, column, nullable) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.AUTO)) { - boolean autoIncrement = line.getOptionValue(GravitinoOptions.AUTO).equals("true"); - newUpdateColumnAutoIncrement( - url, ignore, metalake, catalog, schema, table, column, autoIncrement) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.DEFAULT)) { - String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); - String dataType = line.getOptionValue(GravitinoOptions.DATATYPE); - newUpdateColumnDefault( - url, ignore, metalake, catalog, schema, table, column, defaultValue, dataType) - .validate() - .handle(); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - break; - } - } - private void handleHelpCommand() { String helpFile = entity.toLowerCase() + "_help.txt"; From 0b9d89bba96527037f4bbf76c6fc9f814d5e3660 Mon Sep 17 00:00:00 2001 From: SekiXu Date: Mon, 13 Jan 2025 06:55:42 +0800 Subject: [PATCH 03/17] [#6150] improve(CLI): Refactor user commands in Gavitino CLI (#6193) ### What changes were proposed in this pull request? Refactor user commands and Base class in Gavitino CLI. ### Why are the changes needed? Fix: #6150 ### Does this PR introduce any user-facing change? No ### How was this patch tested? local test. --- .../gravitino/cli/GravitinoCommandLine.java | 63 +------ .../gravitino/cli/UserCommandHandler.java | 174 ++++++++++++++++++ 2 files changed, 175 insertions(+), 62 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index b883502e805..21d3ed176cb 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -143,7 +143,7 @@ private void executeCommand() { } else if (entity.equals(CommandEntities.FILESET)) { handleFilesetCommand(); } else if (entity.equals(CommandEntities.USER)) { - handleUserCommand(); + new UserCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.GROUP)) { new GroupCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TAG)) { @@ -240,67 +240,6 @@ private void handleMetalakeCommand() { } } - /** Handles the command execution for Users based on command type and the command line options. */ - protected void handleUserCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String user = line.getOptionValue(GravitinoOptions.USER); - - Command.setAuthenticationMode(auth, userName); - - if (user == null && !CommandActions.LIST.equals(command)) { - System.err.println(ErrorMessages.MISSING_USER); - Main.exit(-1); - } - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newUserAudit(url, ignore, metalake, user).validate().handle(); - } else { - newUserDetails(url, ignore, metalake, user).validate().handle(); - } - break; - - case CommandActions.LIST: - newListUsers(url, ignore, metalake).validate().handle(); - break; - - case CommandActions.CREATE: - newCreateUser(url, ignore, metalake, user).validate().handle(); - break; - - case CommandActions.DELETE: - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteUser(url, ignore, force, metalake, user).validate().handle(); - break; - - case CommandActions.REVOKE: - String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); - for (String role : revokeRoles) { - newRemoveRoleFromUser(url, ignore, metalake, user, role).validate().handle(); - } - System.out.printf("Remove roles %s from user %s%n", COMMA_JOINER.join(revokeRoles), user); - break; - - case CommandActions.GRANT: - String[] grantRoles = line.getOptionValues(GravitinoOptions.ROLE); - for (String role : grantRoles) { - newAddRoleToUser(url, ignore, metalake, user, role).validate().handle(); - } - System.out.printf("Grant roles %s to user %s%n", COMMA_JOINER.join(grantRoles), user); - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); - Main.exit(-1); - break; - } - } - /** Handles the command execution for Tags based on command type and the command line options. */ protected void handleTagCommand() { String url = getUrl(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java new file mode 100644 index 00000000000..9a8374ec342 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Users based on command type and the command line options. */ +public class UserCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private String user; + + /** + * Constructs a {@link UserCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public UserCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + user = line.getOptionValue(GravitinoOptions.USER); + + if (user == null && !CommandActions.LIST.equals(command)) { + System.err.println(ErrorMessages.MISSING_USER); + Main.exit(-1); + } + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.LIST: + handleListCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.REVOKE: + handleRevokeCommand(); + return true; + + case CommandActions.GRANT: + handleGrantCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + this.gravitinoCommandLine + .newListUsers(this.url, this.ignore, this.metalake) + .validate() + .handle(); + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + this.gravitinoCommandLine + .newUserAudit(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } else { + this.gravitinoCommandLine + .newUserDetails(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + this.gravitinoCommandLine + .newCreateUser(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + this.gravitinoCommandLine + .newDeleteUser(this.url, this.ignore, force, this.metalake, user) + .validate() + .handle(); + } + + /** Handles the "REVOKE" command. */ + private void handleRevokeCommand() { + String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); + for (String role : revokeRoles) { + this.gravitinoCommandLine + .newRemoveRoleFromUser(this.url, this.ignore, this.metalake, user, role) + .validate() + .handle(); + } + System.out.printf("Remove roles %s from user %s%n", COMMA_JOINER.join(revokeRoles), user); + } + + /** Handles the "GRANT" command. */ + private void handleGrantCommand() { + String[] grantRoles = line.getOptionValues(GravitinoOptions.ROLE); + for (String role : grantRoles) { + this.gravitinoCommandLine + .newAddRoleToUser(this.url, this.ignore, this.metalake, user, role) + .validate() + .handle(); + } + System.out.printf("Add roles %s to user %s%n", COMMA_JOINER.join(grantRoles), user); + } +} From bbe3bcf1fdd31ea33a15a940ed57590987250491 Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 13 Jan 2025 08:51:08 +0800 Subject: [PATCH 04/17] [MINOR] bump version to 0.9.0-incubating-snapshot (#6094) ### What changes were proposed in this pull request? bump version to 0.9.0-incubating-snapshot ### Why are the changes needed? change project version after cut branch-0.8 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no, just change version --- clients/client-python/setup.py | 2 +- docs/index.md | 8 ++++---- docs/manage-relational-metadata-using-gravitino.md | 10 +++++----- docs/open-api/openapi.yaml | 2 +- gradle.properties | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index 108ad0226f1..878e74a1d00 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -27,7 +27,7 @@ setup( name="apache-gravitino", description="Python lib/client for Apache Gravitino", - version="0.8.0.dev0", + version="0.9.0.dev0", long_description=long_description, long_description_content_type="text/markdown", author="Apache Software Foundation", diff --git a/docs/index.md b/docs/index.md index 401e6c1d0a9..4a9c43131d9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -61,8 +61,8 @@ REST API and the Java SDK. You can use either to manage metadata. See Also, you can find the complete REST API definition in [Gravitino Open API](./api/rest/gravitino-rest-api), -Java SDK definition in [Gravitino Java doc](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/index.html), -and Python SDK definition in [Gravitino Python doc](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/python/index.html). +Java SDK definition in [Gravitino Java doc](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/index.html), +and Python SDK definition in [Gravitino Python doc](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/python/index.html). Gravitino also provides a web UI to manage the metadata. Visit the web UI in the browser via `http://:8090`. See [Gravitino web UI](./webui.md) for details. @@ -178,8 +178,8 @@ Gravitino provides security configurations for Gravitino, including HTTPS, authe ### Programming guides * [Gravitino Open API](./api/rest/gravitino-rest-api): provides the complete Open API definition of Gravitino. -* [Gravitino Java doc](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/index.html): provides the Javadoc for the Gravitino API. -* [Gravitino Python doc](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/python/index.html): provides the Python doc for the Gravitino API. +* [Gravitino Java doc](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/index.html): provides the Javadoc for the Gravitino API. +* [Gravitino Python doc](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/python/index.html): provides the Python doc for the Gravitino API. ### Development guides diff --git a/docs/manage-relational-metadata-using-gravitino.md b/docs/manage-relational-metadata-using-gravitino.md index 352a8de2935..b3d28e95128 100644 --- a/docs/manage-relational-metadata-using-gravitino.md +++ b/docs/manage-relational-metadata-using-gravitino.md @@ -909,7 +909,7 @@ The following types that Gravitino supports: | Union | `Types.UnionType.of([type1, type2, ...])` | `{"type": "union", "types": [type JSON, ...]}` | Union type, indicates a union of types | | UUID | `Types.UUIDType.get()` | `uuid` | UUID type, indicates a universally unique identifier | -The related java doc is [here](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/types/Type.html). +The related java doc is [here](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/types/Type.html). ##### External type @@ -1022,10 +1022,10 @@ In addition to the basic settings, Gravitino supports the following features: | Feature | Description | Java doc | |---------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| -| Table partitioning | Equal to `PARTITION BY` in Apache Hive, It is a partitioning strategy that is used to split a table into parts based on partition keys. Some table engine may not support this feature | [Partition](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/dto/rel/partitioning/Partitioning.html) | -| Table distribution | Equal to `CLUSTERED BY` in Apache Hive, distribution a.k.a (Clustering) is a technique to split the data into more manageable files/parts, (By specifying the number of buckets to create). The value of the distribution column will be hashed by a user-defined number into buckets. | [Distribution](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/expressions/distributions/Distribution.html) | -| Table sort ordering | Equal to `SORTED BY` in Apache Hive, sort ordering is a method to sort the data in specific ways such as by a column or a function, and then store table data. it will highly improve the query performance under certain scenarios. | [SortOrder](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/expressions/sorts/SortOrder.html) | -| Table indexes | Equal to `KEY/INDEX` in MySQL , unique key enforces uniqueness of values in one or more columns within a table. It ensures that no two rows have identical values in specified columns, thereby facilitating data integrity and enabling efficient data retrieval and manipulation operations. | [Index](pathname:///docs/0.8.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/indexes/Index.html) | +| Table partitioning | Equal to `PARTITION BY` in Apache Hive, It is a partitioning strategy that is used to split a table into parts based on partition keys. Some table engine may not support this feature | [Partition](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/dto/rel/partitioning/Partitioning.html) | +| Table distribution | Equal to `CLUSTERED BY` in Apache Hive, distribution a.k.a (Clustering) is a technique to split the data into more manageable files/parts, (By specifying the number of buckets to create). The value of the distribution column will be hashed by a user-defined number into buckets. | [Distribution](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/expressions/distributions/Distribution.html) | +| Table sort ordering | Equal to `SORTED BY` in Apache Hive, sort ordering is a method to sort the data in specific ways such as by a column or a function, and then store table data. it will highly improve the query performance under certain scenarios. | [SortOrder](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/expressions/sorts/SortOrder.html) | +| Table indexes | Equal to `KEY/INDEX` in MySQL , unique key enforces uniqueness of values in one or more columns within a table. It ensures that no two rows have identical values in specified columns, thereby facilitating data integrity and enabling efficient data retrieval and manipulation operations. | [Index](pathname:///docs/0.9.0-incubating-SNAPSHOT/api/java/org/apache/gravitino/rel/indexes/Index.html) | For more information, please see the related document on [partitioning, bucketing, sorting, and indexes](table-partitioning-bucketing-sort-order-indexes.md). diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index f39a90f55f5..4405f130135 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -22,7 +22,7 @@ info: license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - version: 0.8.0-incubating-SNAPSHOT + version: 0.9.0-incubating-SNAPSHOT description: | Defines the specification for the first version of the Gravitino REST API. diff --git a/gradle.properties b/gradle.properties index cc1b9393018..4049f73840b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,7 +23,7 @@ org.gradle.caching=true org.gradle.jvmargs=-Xmx4g # version that is going to be updated automatically by releases -version = 0.8.0-incubating-SNAPSHOT +version = 0.9.0-incubating-SNAPSHOT # sonatype credentials SONATYPE_USER = admin From 2d0cda5c43215688e7106892ae3f9d5bbbefbe87 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Mon, 13 Jan 2025 13:13:59 +1100 Subject: [PATCH 05/17] [#6194] Add python client license and notice file (#6195) ## What changes were proposed in this pull request? Add license and notice files. ### Why are the changes needed? As the release's content is different to that of Gravitino. Fix: #6194 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Tested locally. --- clients/client-python/LICENSE | 214 +++++++++++++++++++++ clients/client-python/NOTICE | 8 + clients/client-python/build.gradle.kts | 6 - clients/client-python/licenses/kylinpy.txt | 21 ++ 4 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 clients/client-python/LICENSE create mode 100644 clients/client-python/NOTICE create mode 100644 clients/client-python/licenses/kylinpy.txt diff --git a/clients/client-python/LICENSE b/clients/client-python/LICENSE new file mode 100644 index 00000000000..42c856d10b1 --- /dev/null +++ b/clients/client-python/LICENSE @@ -0,0 +1,214 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Web UI also bundles various third-party components also under + different licenses, please see web/LICENSE for these. + + This product bundles various third-party components also under the + Apache Software License 2.0. + + This product bundles a third-party component under the + MIT License. + + Kyligence/kylinpy + ./client-python/gravitino/utils/http_client.py + diff --git a/clients/client-python/NOTICE b/clients/client-python/NOTICE new file mode 100644 index 00000000000..c1fde5e04e3 --- /dev/null +++ b/clients/client-python/NOTICE @@ -0,0 +1,8 @@ +Apache Gravitino (incubating) +Copyright 2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The initial code for the Gravitino project was donated +to the ASF by Datastrato (https://datastrato.ai/) copyright 2023-2024. \ No newline at end of file diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index bebf536f6eb..af6cfcd2d9f 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -285,9 +285,6 @@ tasks { generatePypiProjectHomePage() delete("dist") copy { - from("${project.rootDir}/licenses") { into("licenses") } - from("${project.rootDir}/LICENSE.bin") { into("./") } - from("${project.rootDir}/NOTICE.bin") { into("./") } from("${project.rootDir}/DISCLAIMER_WIP.txt") { into("./") } into("${project.rootDir}/clients/client-python") rename { fileName -> @@ -301,9 +298,6 @@ tasks { doLast { delete("README.md") - delete("licenses") - delete("LICENSE") - delete("NOTICE") delete("DISCLAIMER_WIP.txt") } } diff --git a/clients/client-python/licenses/kylinpy.txt b/clients/client-python/licenses/kylinpy.txt new file mode 100644 index 00000000000..580127c7327 --- /dev/null +++ b/clients/client-python/licenses/kylinpy.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Dhamu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From 9815cc3690652a7df06d070bf0b978b89755997d Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:37:54 +0800 Subject: [PATCH 06/17] [#6069] fix(docs): Fix access-control.md (#6189) ### What changes were proposed in this pull request? Fix the wrong document information about revoke roles from role ### Why are the changes needed? Fix: #6069 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? local test. --- docs/security/access-control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/security/access-control.md b/docs/security/access-control.md index 7e996738cb6..681ec4752d5 100644 --- a/docs/security/access-control.md +++ b/docs/security/access-control.md @@ -817,7 +817,7 @@ curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ ```java GravitinoClient client = ... -Group group = client.grantRolesToGroup(Lists.newList("role1"), "group1"); +Group group = client.revokeRolesFromGroup(Lists.newList("role1"), "group1"); ``` From d9ae375d211c64ae6318c32add11e476c901c5f7 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Mon, 13 Jan 2025 11:15:02 +0800 Subject: [PATCH 07/17] [#5533] fix (trino-connector): Fix the exception of ArrayIndexOutOfBoundsException when execute COMMENT COLUMN command (#6182) ### What changes were proposed in this pull request? Fix the exception of ArrayIndexOutOfBoundsException when handle error message of IllegalArgumentException ### Why are the changes needed? Fix: #5533 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? IT --- .../integration/test/TrinoQueryIT.java | 24 +++++++++---------- .../integration/test/TrinoQueryRunner.java | 19 ++++++++------- .../testsets/jdbc-mysql/00002_alter_table.sql | 2 ++ .../testsets/jdbc-mysql/00002_alter_table.txt | 2 ++ .../trino/connector/GravitinoErrorCode.java | 6 +++++ .../catalog/CatalogConnectorMetadata.java | 3 +-- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java index d9940de4573..64e49723a6e 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java @@ -55,15 +55,15 @@ public class TrinoQueryIT extends TrinoQueryITBase { private static final Logger LOG = LoggerFactory.getLogger(TrinoQueryIT.class); - static String testsetsDir = ""; - AtomicInteger passCount = new AtomicInteger(0); - AtomicInteger totalCount = new AtomicInteger(0); - static boolean exitOnFailed = true; + protected static String testsetsDir; + protected AtomicInteger passCount = new AtomicInteger(0); + protected AtomicInteger totalCount = new AtomicInteger(0); + protected static boolean exitOnFailed = true; // key: tester name, value: tester result - private static Map allTestStatus = new TreeMap<>(); + private static final Map allTestStatus = new TreeMap<>(); - private static int testParallelism = 2; + private static final int testParallelism = 2; static Map queryParams = new HashMap<>(); @@ -275,8 +275,8 @@ void executeSqlFileWithCheckResult( * actual result matches the query failed result. 3. The expected result is a regular expression, * and the actual result matches the regular expression. * - * @param expectResult - * @param result + * @param expectResult the expected result + * @param result the actual result * @return false if the expected result is empty or the actual result does not match the expected. * For {@literal } case, return true if the actual result is empty. For {@literal * } case, replace the placeholder with "^Query \\w+ failed.*: " and do match. @@ -338,7 +338,7 @@ static boolean match(String expectResult, String result) { @Test public void testSql() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(testParallelism); - CompletionService completionService = new ExecutorCompletionService<>(executor); + CompletionService completionService = new ExecutorCompletionService<>(executor); String[] testSetNames = Arrays.stream(TrinoQueryITBase.listDirectory(testsetsDir)) @@ -357,7 +357,7 @@ public void testSql() throws Exception { public void testSql(String testSetDirName, String catalog, String testerPrefix) throws Exception { ExecutorService executor = Executors.newFixedThreadPool(testParallelism); - CompletionService completionService = new ExecutorCompletionService<>(executor); + CompletionService completionService = new ExecutorCompletionService<>(executor); totalCount.addAndGet(getTesterCount(testSetDirName, catalog, testerPrefix)); List> futures = @@ -369,7 +369,7 @@ public void testSql(String testSetDirName, String catalog, String testerPrefix) private void waitForCompleted( ExecutorService executor, - CompletionService completionService, + CompletionService completionService, List> allFutures) { for (int i = 0; i < allFutures.size(); i++) { try { @@ -405,7 +405,7 @@ public String generateTestStatus() { } public List> runOneTestset( - CompletionService completionService, + CompletionService completionService, String testSetDirName, String catalog, String testerFilter) diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryRunner.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryRunner.java index 0e794e45ab5..7c3001a731e 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryRunner.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryRunner.java @@ -42,9 +42,9 @@ class TrinoQueryRunner { private static final Logger LOG = LoggerFactory.getLogger(TrinoQueryRunner.class); - private QueryRunner queryRunner; - private Terminal terminal; - private URI uri; + private final QueryRunner queryRunner; + private final Terminal terminal; + private final URI uri; TrinoQueryRunner(String trinoUri) throws Exception { this.uri = new URI(trinoUri); @@ -92,10 +92,11 @@ String runQuery(String query) { String runQueryOnce(String query) { Query queryResult = queryRunner.startQuery(query); StringOutputStream outputStream = new StringOutputStream(); + StringOutputStream errorStream = new StringOutputStream(); queryResult.renderOutput( this.terminal, new PrintStream(outputStream), - new PrintStream(outputStream), + new PrintStream(errorStream), CSV, Optional.of(""), false); @@ -109,17 +110,19 @@ String runQueryOnce(String query) { session = builder.build(); queryRunner.setSession(session); } - return outputStream.toString(); + + // Avoid the IDE capturing the error message as failure + String err_message = errorStream.toString().replace("\nCaused by:", "\n-Caused by:"); + String out_message = outputStream.toString(); + return err_message + out_message; } - boolean stop() { + void stop() { try { queryRunner.close(); terminal.close(); - return true; } catch (Exception e) { LOG.error("Failed to stop query runner", e); - return false; } } } diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql index b3af09a6580..e8058cde4ef 100644 --- a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql +++ b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.sql @@ -37,6 +37,8 @@ show create table gt_mysql.gt_db1.tb01; alter table gt_mysql.gt_db1.tb01 add column address varchar(200) not null comment 'address of users'; show create table gt_mysql.gt_db1.tb01; +COMMENT ON COLUMN gt_mysql.gt_db1.tb01.city IS NULL; + drop table gt_mysql.gt_db1.tb01; drop schema gt_mysql.gt_db1; diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt index 3aa3144935c..b3b5366b9a6 100644 --- a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt +++ b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00002_alter_table.txt @@ -104,6 +104,8 @@ WITH ( engine = 'InnoDB' )" + "newComment" field is required and cannot be empty + DROP TABLE DROP SCHEMA diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoErrorCode.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoErrorCode.java index 5741e4427bd..e47675d4574 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoErrorCode.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/GravitinoErrorCode.java @@ -23,6 +23,7 @@ import io.trino.spi.ErrorCode; import io.trino.spi.ErrorCodeSupplier; import io.trino.spi.ErrorType; +import java.util.List; public enum GravitinoErrorCode implements ErrorCodeSupplier { GRAVITINO_UNSUPPORTED_TRINO_VERSION(0, EXTERNAL), @@ -64,4 +65,9 @@ public enum GravitinoErrorCode implements ErrorCodeSupplier { public ErrorCode toErrorCode() { return errorCode; } + + public static String toSimpleErrorMessage(Exception e) { + List lines = e.getMessage().lines().toList(); + return lines.size() > 1 ? lines.get(0) + lines.get(1) : lines.get(0); + } } diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java index 759a4de0889..3bb61f977e5 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java @@ -190,8 +190,7 @@ private void applyAlter(SchemaTableName tableName, TableChange... change) { // TODO yuhui need improve get the error message. From IllegalArgumentException. // At present, the IllegalArgumentException cannot get the error information clearly from the // Gravitino server. - String message = - e.getMessage().lines().toList().get(0) + e.getMessage().lines().toList().get(1); + String message = GravitinoErrorCode.toSimpleErrorMessage(e); throw new TrinoException(GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT, message, e); } } From 1fa31013a40b7a2d98988b29370e12a6b7abf94e Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Mon, 13 Jan 2025 15:12:34 +1100 Subject: [PATCH 08/17] [Minor] Update command usage and add usage tracker in Gravitino CLI (#6137) ### What changes were proposed in this pull request? Update command usage and add usage tracker ### Why are the changes needed? So everything is up to date and we can see many many people look up the CLI docs. Fix: # N/A ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Tested locally. --- docs/cli.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 0cc7dee4af9..0598a36e034 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -30,13 +30,17 @@ The general structure for running commands with the Gravitino CLI is `gcli entit usage: gcli [metalake|catalog|schema|model|table|column|user|group|tag|topic|fileset] [list|details|create|delete|update|set|remove|properties|revoke|grant] [options] Options usage: gcli - -a,--audit display audit information + -a,--audit display audit information + --alias model aliases + --all all operation for --enable --auto column value auto-increments (true/false) -c,--comment entity comment --columnfile CSV file describing columns -d,--distribution display distribution information --datatype column data type --default default column value + --disable disable entities + --enable enable entities -f,--force force operation -g,--group group name -h,--help command help information @@ -52,6 +56,7 @@ The general structure for running commands with the Gravitino CLI is `gcli entit -p,--properties property name/value pairs --partition display partition information --position position of column + --privilege privilege(s) -r,--role role name --rename new entity name -s,--server Gravitino server version @@ -59,6 +64,7 @@ The general structure for running commands with the Gravitino CLI is `gcli entit --sortorder display sortorder information -t,--tag tag name -u,--url Gravitino URL (default: http://localhost:8090) + --uri model version artifact -v,--version Gravitino client version -V,--value property value -x,--index display index information @@ -950,4 +956,6 @@ gcli --simple ```bash gcli --simple --login userName -``` \ No newline at end of file +``` + + \ No newline at end of file From b2b2338d52003eceaa2e8ee959b73baf5c32c72a Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Mon, 13 Jan 2025 13:54:20 +0800 Subject: [PATCH 09/17] [doc] Revise the glossary documentation (#5837) ### What changes were proposed in this pull request? This PR fixes the glossary docs. ### Why are the changes needed? The glossary is reordered for quick reference. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? N/A --- docs/glossary.md | 384 ++++++++++++++++++++++++++++------------------- 1 file changed, 226 insertions(+), 158 deletions(-) diff --git a/docs/glossary.md b/docs/glossary.md index 83e97d915aa..3b42a6c7734 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -4,41 +4,180 @@ date: 2023-11-28 license: "This software is licensed under the Apache License version 2." --- +## API + +- Application Programming Interface, defining the methods and protocols for interacting with a server. + +## AWS + +- Amazon Web Services, a cloud computing platform provided by Amazon. + +## AWS Glue + +- A compatible implementation of the Hive Metastore Service (HMS). + +## GPG/GnuPG + +- Gnu Privacy Guard or GnuPG is an open-source implementation of the OpenPGP standard. + It is usually used for encrypting and signing files and emails. + +## HDFS + +- **HDFS** (Hadoop Distributed File System) is an open-source distributed file system. + It is a key component of the Apache Hadoop ecosystem. + HDFS is designed as a distributed storage solution to store and process large-scale datasets. + It features high reliability, fault tolerance, and excellent performance. + +## HTTP port + +- The port number on which a server listens for incoming connections. + +## IP address + +- Internet Protocol address, a numerical label assigned to each device in a computer network. + +## JDBC + +- Java Database Connectivity, an API for connecting Java applications to relational databases. + +## JDBC URI + +- The JDBC connection address specified in the catalog configuration. + It usually includes components such as the database type, host, port, and database name. + +## JDK + +- The software development kit for the Java programming language. + A JDK provides tools for compiling, debugging, and running Java applications. + +## JMX + +- Java Management Extensions provides tools for managing and monitoring Java applications. + +## JSON + +- JavaScript Object Notation, a lightweight data interchange format. + +## JSON Web Token + +- See [JWT](#jwt). + +## JVM + +- A virtual machine that enables a computer to run Java applications. + A JVM implements an abstract machine that is different from the underlying hardware. + +## JVM instrumentation + +- The process of adding monitoring and management capabilities to the [JVM](#jvm). + The purpose of instrumentation is mainly for the collection of performance metrics. + +## JVM metrics + +- Metrics related to the performance and behavior of the [Java Virtual Machine](#jvm). + Some valuable metrics are memory usage, garbage collection, and buffer pool metrics. + +## JWT + +- A compact, URL-safe representation for claims between two parties. + +## KEYS file + +- A file containing public keys used to sign previous releases, necessary for verifying signatures. + +## PGP signature + +- A digital signature generated using the Pretty Good Privacy (PGP) algorithm. + The signature is typically used to validate the authenticity of a file. + +## REST + +- A set of architectural principles for designing networked applications. + +## REST API + +- Representational State Transfer (REST) Application Programming Interface. + A set of rules and conventions for building and interacting with Web services using standard HTTP methods. + +## SHA256 checksum + +- A cryptographic hash function used to verify the integrity of files. + +## SHA256 checksum file + +- A file containing the SHA256 hash value of another file, used for verification purposes. + +## SQL + +- A programming language used to manage and manipulate relational databases. + +## SSH + +- Secure Shell, a cryptographic network protocol used for secure communication over a computer network. + +## URI + +- Uniform Resource Identifier, a string that identifies the name or resource on the internet. + +## YAML + +- YAML Ain't Markup Language, a human-readable file format often used for structured data. + +## Amazon Elastic Block Store (EBS) + +- A scalable block storage service provided by Amazon Web Services (AWS). + +## Apache Gravitino + +- An open-source software platform initially created by Datastrato. + It is designed for high-performance, geo-distributed, and federated metadata lakes. + Gravitino can manage metadata directly in different sources, types, and regions, + providing data and AI assets with unified metadata access. + +## Apache Gravitino configuration file (gravitino.conf) + +- The configuration file for the Gravitino server, located in the `conf` directory. + It follows the standard properties file format and contains settings for the Gravitino server. + ## Apache Hadoop - An open-source distributed storage and processing framework. ## Apache Hive -- An open-source data warehousing and SQL-like query language software project for managing and querying large datasets. +- An open-source data warehousing software project. + It provides SQL-like query language for managing and querying large datasets. ## Apache Iceberg - An open-source, versioned table format for large-scale data processing. -## Apache License version 2 +## Apache Iceberg Hive catalog -- A permissive, open-source software license written by The Apache Software Foundation. +- The **Iceberg Hive catalog** is a metadata service designed for the Apache Iceberg table format. + It allows external systems to interact with an Iceberg metadata using a Hive metastore thrift client. -## API +## Apache Iceberg JDBC catalog -- Application Programming Interface, defining the methods and protocols for interacting with a server. +- The **Iceberg JDBC catalog** is a metadata service designed for the Apache Iceberg table format. + It enables external systems to interact with an Iceberg metadata service using [JDBC](#jdbc). -## Authentication mechanism +## Apache Iceberg REST catalog -- The method used to verify the identity of users and clients accessing a server. +- The **Iceberg REST Catalog** is a metadata service designed for the Apache Iceberg table format. + It enables external systems to interact with Iceberg metadata service using a [REST API](#rest-api). -## AWS +## Apache License version 2 -- Amazon Web Services, a cloud computing platform provided by Amazon. +- A permissive, open-source software license written by The Apache Software Foundation. -## AWS Glue +## Authentication mechanism -- A compatible implementation of the Hive Metastore Service (HMS). +- The method used to verify the identity of users and clients accessing a server. ## Binary distribution package -- A package containing the compiled and executable version of the software, ready for distribution and deployment. +- A software package containing the compiled executables for distribution and deployment. ## Catalog @@ -50,15 +189,12 @@ license: "This software is licensed under the Apache License version 2." ## Columns -- The individual fields or attributes of a table, specifying details such as name, data type, comment, and nullability. +- The individual fields or attributes of a table. + Each column has properties like name, data type, comment, and nullability. ## Continuous integration (CI) -- The practice of automatically building, testing, and validating code changes when they are committed to version control. - -## Contributor covenant - -- A widely-used and recognized code of conduct for open-source communities. It provides guidelines for creating a welcoming and inclusive environment for all contributors. +- The practice of automatically building and testing code changes when they are committed to version control. ## Dependencies @@ -74,51 +210,56 @@ license: "This software is licensed under the Apache License version 2." ## Docker container -- A lightweight, standalone, executable package that includes everything needed to run a piece of software, including the code, runtime, libraries, and system tools. +- A lightweight, standalone package that includes everything needed to run the software. + A container compiles an application with its dependencies and runtime for distribution. ## Docker Hub -- A cloud-based registry service for Docker containers, allowing users to share and distribute containerized applications. +- A cloud-based registry service for Docker containers. + Users can publish, browse and download containerized software using this service. ## Docker image -- A lightweight, standalone, and executable package that includes everything needed to run a piece of software, including the code, runtime, libraries, and system tools. +- A lightweight, standalone package that includes everything needed to run the software. + A Docker image typically comprises the code, runtime, libraries, and system tools. -## Docker file +## Dockerfile -- A configuration file used to create a Docker image, specifying the base image, dependencies, and commands for building the image. +- A configuration file for building a Docker image. + A Dockerfile contains instructions to build a standard image for distributing the software. -## Dropwizard Metrics +## Dropwizard metrics - A Java library for measuring the performance of applications and providing support for various metric types. -## Amazon Elastic Block Store (EBS) - -- A scalable block storage service provided by Amazon Web Services. - ## Environment variables -- Variables used to pass information to running processes. +- Variables used to customize the runtime configuration for a process. ## Geo-distributed - The distribution of data or services across multiple geographic locations. +## Git + +- A distributed version control system used for tracking software artifacts. + ## GitHub -- A web-based platform for version control and collaboration using Git. +- A web-based platform for version control and community collaboration using Git. ## GitHub Actions -- A continuous integration and continuous deployment (CI/CD) service provided by GitHub, used for automating build, test, and deployment workflows. +- A continuous integration and continuous deployment (CI/CD) service provided by GitHub. + GitHub Actions automate the build, test, and deployment workflows. ## GitHub labels -- Tags assigned to GitHub issues or pull requests for organization, categorization, or workflow automation. +- Labels assigned to GitHub issues or pull requests for organization or workflow automation. ## GitHub pull request -- A proposed change to a repository submitted by a user through the GitHub platform. +- A proposed change to a GitHub repository submitted by a user. ## GitHub repository @@ -126,127 +267,67 @@ license: "This software is licensed under the Apache License version 2." ## GitHub workflow -- A series of automated steps defined in a YAML file that runs in response to events on a GitHub repository. - -## Git - -- A version control system used for tracking changes and collaborating on source code. - -## GPG/GnuPG - -- Gnu Privacy Guard or GnuPG, an open-source implementation of the OpenPGP standard, used for encrypting and signing files and emails. +- A series of automated steps triggered by specific events on a GitHub repository. ## Gradle -- A build automation tool for building, testing, and deploying projects. +- An automation tool for building, testing, and deploying projects. ## Gradlew -- A Gradle wrapper script, used for executing Gradle commands without installing Gradle separately. - -## Apache Gravitino - -- An open-source software platform originally created by Datastrato for high-performance, geo-distributed, and federated metadata lakes. Designed to manage metadata directly in different sources, types, and regions, providing unified metadata access for data and AI assets. - -## Apache Gravitino configuration file (gravitino.conf) - -- The configuration file for the Gravitino server, located in the `conf` directory. It follows the standard property file format and contains settings for the Gravitino server. +- A Gradle wrapper script used to execute Gradle commands. ## Hashes -- Cryptographic hash values generated from the contents of a file, often used for integrity verification. - -## HDFS - -- **HDFS** (Hadoop Distributed File System) is an open-source, distributed file system and a key component of the Apache Hadoop ecosystem. It is designed to store and process large-scale datasets, providing high reliability, fault tolerance, and performance for distributed storage solutions. +- Cryptographic hash values generated from some data. + A typical use case is to verify the integrity of a file. ## Headless -- A system without a graphical user interface. - -## HTTP port - -- The port number on which a server listens for incoming connections. - -## Apache Iceberg Hive catalog - -- The **Iceberg Hive catalog** is a specialized metadata service designed for the Apache Iceberg table format, allowing external systems to interact with Iceberg metadata via a Hive metastore thrift client. - -## Apache Iceberg REST catalog - -- The **Iceberg REST Catalog** is a specialized metadata service designed for the Apache Iceberg table format, allowing external systems to interact with Iceberg metadata via a RESTful API. - -## Apache Iceberg JDBC catalog - -- The **Iceberg JDBC Catalog** is a specialized metadata service designed for the Apache Iceberg table format, allowing external systems to interact with Iceberg metadata using JDBC (Java Database Connectivity). +- A system without a local console. ## Identity fields -- Fields in tables that define the identity of the table, specifying how rows in the table are uniquely identified. +- Fields in tables that define the identity of the records. + In the scope of a table, the identity fields are used as the unique identifier of a row. ## Integration tests -- Tests designed to ensure the correctness and compatibility of software when integrated into a unified system. - -## IP address - -- Internet Protocol address, a numerical label assigned to each device participating in a computer network. +- Tests that ensure software correctness and compatibility when integrating components into a larger system. ## Java Database Connectivity (JDBC) -- Java Database Connectivity, an API for connecting Java applications to relational databases. +- See [JDBC](#jdbc) ## Java Development Kits (JDKs) -- Software development kits for the Java programming language, including tools for compiling, debugging, and running Java applications. - -## Java Toolchain +- See [JDK](#jdk) -- A feature introduced in Gradle to detect and manage JDK versions. +## Java Management Extensions -## JDBC URI - -- The JDBC connection address specified in the catalog configuration, including details such as the database type, host, port, and database name. - -## JMX - -- Java Management Extensions provides tools for managing and monitoring Java applications. - -## JSON - -- JavaScript Object Notation, a lightweight data interchange format. +- See [JMX](#jmx) -## JWT(JSON Web Token) - -- A compact, URL-safe means of representing claims between two parties. - -## Java Virtual Machine (JVM) - -- A virtual machine that enables a computer to run Java applications, providing an abstraction layer between the application and the underlying hardware. - -## JVM metrics +## Java Toolchain -- Metrics related to the performance and behavior of the Java Virtual Machine (JVM), including memory usage, garbage collection, and buffer pool metrics. +- A Gradle feature for detecting and managing JDK versions. -## JVM instrumentation +## Java Virtual Machine -- The process of adding monitoring and management capabilities to the Java Virtual Machine, allowing for the collection of performance metrics. +- See [JVM](#jvm) ## Key pair - A pair of cryptographic keys, including a public key used for verification and a private key used for signing. -## KEYS file - -- A file containing public keys used to sign previous releases, necessary for verifying signatures. - ## Lakehouse -- **Lakehouse** refers to a modern data management architecture that combines elements of data lakes and data warehouses. It aims to provide a unified platform for storing, managing, and analyzing both raw unstructured data (similar to data lakes) and curated structured data. +- **Lakehouse** is a modern data management architecture that combines elements of data lakes and data warehouses. + It aims to provide a unified platform for storing, managing, and analyzing both raw unstructured data + (similar to data lakes) and curated structured data. ## Manifest -- A list of files and associated metadata that collectively define the structure and content of a release or distribution. +- A list of files and their associated metadata that collectively define the structure and content of a release or distribution. ## Merge operation @@ -254,7 +335,9 @@ license: "This software is licensed under the Apache License version 2." ## Metalake -- The top-level container for metadata. Typically, a metalake is a tenant-like mapping to an organization or a company. All the catalogs, users, and roles are under one metalake. +- The top-level container for metadata. + Typically, a metalake is a tenant-like mapping to an organization or a company. + All the catalogs, users, and roles are associated with one metalake. ## Metastore @@ -264,17 +347,14 @@ license: "This software is licensed under the Apache License version 2." - A distinct and separable part of a project. -## OrbStack - -- A tool mentioned as an alternative to Docker for macOS when running Gravitino integration tests. - ## Open authorization / OAuth -- A standard protocol for authorization that allows third-party applications to access user data without exposing user credentials. +- A standard protocol for authorization that allows third-party applications to authenticate a user. + The application doesn't need to access the user credentials. -## PGP Signature +## OrbStack -- A digital signature generated using the Pretty Good Privacy (PGP) algorithm, confirming the authenticity of a file. +- A tool mentioned as an alternative to Docker for macOS when running Gravitino integration tests. ## Private key @@ -282,31 +362,33 @@ license: "This software is licensed under the Apache License version 2." ## Properties -- Configurable settings and attributes associated with catalogs, schemas, and tables, to influence their behavior and storage. +- Configurable settings and attributes associated with catalogs, schemas, and tables. + The property settings influence the behavior and storage of the corresponding entities. ## Protocol buffers (protobuf) -- A method developed by Google for serializing structured data, similar to XML or JSON. It is often used for efficient and extensible communication between systems. +- A method developed by Google for serializing structured data, similar to XML or JSON. + It is often used for efficient and extensible communication between systems. ## Public key - An openly shared key used for verification, encryption, or other operations intended for public knowledge. -## Representational State Transfer (REST) +## Representational State Transfer -- A set of architectural principles for designing networked applications. +- See [REST](#rest) -## REST API (Representational State Transfer Application Programming Interface) +## RocksDB -- A set of rules and conventions for building and interacting with web services using standard HTTP methods. +- An open source key-value storage database. ## Schema - A logical container for organizing tables in a database. -## Secure Shell (SSH) +## Secure Shell -- Secure Shell, a cryptographic network protocol used for secure communication over a computer network. +- See [SSH](#ssh) ## Security group @@ -314,15 +396,8 @@ license: "This software is licensed under the Apache License version 2." ## Serde -- A Serialization/Deserialization library responsible for transforming data between a tabular format and a format suitable for storage or transmission. - -## SHA256 checksum - -- A cryptographic hash function used to verify the integrity of files. - -## SHA256 checksum file - -- A file containing the SHA256 hash value of another file, used for verification purposes. +- A serialization/deserialization library. + It can transform data between a tabular format and a format suitable for storage or transmission. ## Snapshot @@ -336,21 +411,22 @@ license: "This software is licensed under the Apache License version 2." - A tool or process used to enforce code formatting standards and apply automatic formatting to code. -## Structured Query Language (SQL) +## Structured Query Language -- A programming language used to manage and manipulate relational databases. +- See [SQL](#sql) ## Table - A structured set of data elements stored in columns and rows. -## Token +## Thrift -- A **token** in the context of computing and security commonly refers to a small, indivisible unit of data. Tokens play a crucial role in various domains, including authentication, authorization, and cryptographic systems. +- A network protocol used for communication with Hive Metastore Service (HMS). -## Thrift protocol +## Token -- The network protocol used for communication with Hive Metastore Service (HMS). +- A **token** in the context of computing and security is a small, indivisible unit of data. + Tokens play a crucial role in various domains, including authentication and authorization. ## Trino @@ -360,30 +436,22 @@ license: "This software is licensed under the Apache License version 2." - A connector module for integrating Gravitino with Trino. -## Trino Apache Gravitino connector documentation - -- Documentation providing information on using the Trino connector to access metadata in Gravitino. - ## Ubuntu - A Linux distribution based on Debian, widely used for cloud computing and servers. ## Unit test -- A type of testing where individual components or functions of a program are tested to ensure they work as expected in isolation. - -## URI - -- Uniform Resource Identifier, a string that identifies the name or resource on the internet. +- A type of software testing where individual components or functions of a program are tested. + Unit tests help to ensure that the component or function works as expected in isolation. ## Verification -- The process of confirming the authenticity and integrity of a release by checking its signature and associated hashes. +- The process of confirming the authenticity and integrity of a release. + This is usually done by checking its signature and associated hash values. -## WEB UI +## Web UI - A graphical interface accessible through a web browser. -## YAML -- YAML Ain't Markup Language, a human-readable data serialization format often used for configuration files. From 1e97f475970b7a446b16bea7c0c453dfff8a504d Mon Sep 17 00:00:00 2001 From: roryqi Date: Mon, 13 Jan 2025 14:50:10 +0800 Subject: [PATCH 10/17] [#6200] improvement(docs): Add Docker image details for 0.8.0 (#6202) ### What changes were proposed in this pull request? Add Docker image details for 0.8.0 ### Why are the changes needed? Fix: #6200 ### Does this PR introduce _any_ user-facing change? Add doc. ### How was this patch tested? No need. --- docs/docker-image-details.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index 48b3bd191a1..a137923a694 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -19,6 +19,10 @@ docker run --rm -d -p 8090:8090 -p 9001:9001 apache/gravitino:0.7.0-incubating Changelog + +- apache/gravitino:0.8.0-incubating + - Based on Gravitino 0.8.0-incubating, you can know more information from 0.8.0-incubating [release notes](https://github.com/apache/gravitino/releases/tag/v0.8.0-incubating). + - apache/gravitino:0.7.0-incubating - Based on Gravitino 0.7.0-incubating, you can know more information from 0.7.0-incubating [release notes](https://github.com/apache/gravitino/releases/tag/v0.7.0-incubating). - Place bundle jars (gravitino-aws-bundle.jar, gravitino-gcp-bundle.jar, gravitino-aliyun-bundle.jar) in the `${GRAVITINO_HOME}/catalogs/hadoop/libs` folder to support the cloud storage catalog without manually adding the jars to the classpath. @@ -62,6 +66,12 @@ Changelog - apache/gravitino-iceberg-rest:0.8.0-incubating - Supports OSS and ADLS storage. + +- apache/gravitino-iceberg-rest:0.8.0-incubating + - Supports OSS and ADLS storage. + - Supports event listener. + - Supports audit log. + - apache/gravitino-iceberg-rest:0.7.0-incubating - Using JDBC catalog backend. - Supports S3 and GCS storage. @@ -100,10 +110,14 @@ Changelog ### Trino image Changelog + + +- apache/gravitino-playground:trino-435-gravitino-0.8.0-incubating + - Use Gravitino release 0.8.0-incubating Dockerfile to build the image. + - apache/gravitino-playground:trino-435-gravitino-0.7.0-incubating - Use Gravitino release 0.7.0-incubating Dockerfile to build the image. -Changelog - apache/gravitino-playground:trino-435-gravitino-0.6.1-incubating - Use Gravitino release 0.6.1-incubating Dockerfile to build the image. From d32af61bc56d902ce066cf96dde0449923a6aea5 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 13 Jan 2025 14:58:57 +0800 Subject: [PATCH 11/17] [#5545] fix(doris-catalog): Fix the problem that we can't set Doris table properties. (#6186) ### What changes were proposed in this pull request? Modify table properties SQL in alter table sentence to support setting table properties. ### Why are the changes needed? It's a bug. Fix: #5545 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? IT. --- .../doris/operation/DorisTableOperations.java | 17 ++++++++--------- .../doris/integration/test/CatalogDorisIT.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java index aa6348e2f71..829088f0131 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -567,10 +567,6 @@ protected String generateAlterTableSql( alterSql.add("MODIFY COMMENT \"" + newComment + "\""); } - if (!setProperties.isEmpty()) { - alterSql.add(generateTableProperties(setProperties)); - } - if (CollectionUtils.isEmpty(alterSql)) { return ""; } @@ -602,11 +598,14 @@ private String updateColumnNullabilityDefinition( } private String generateTableProperties(List setProperties) { - return setProperties.stream() - .map( - setProperty -> - String.format("\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue())) - .collect(Collectors.joining(",\n")); + String properties = + setProperties.stream() + .map( + setProperty -> + String.format( + "\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue())) + .collect(Collectors.joining(",\n")); + return "set (" + properties + ")"; } private String updateColumnCommentFieldDefinition( diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 9288c9616bc..9d2c798ae7e 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -577,6 +577,16 @@ void testAlterDorisTable() { .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) .untilAsserted( () -> assertEquals(4, tableCatalog.loadTable(tableIdentifier).columns().length)); + + // set property + tableCatalog.alterTable(tableIdentifier, TableChange.setProperty("in_memory", "true")); + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertEquals( + "true", tableCatalog.loadTable(tableIdentifier).properties().get("in_memory"))); } @Test From 7de40b88c6aa0124edc42eb44c460bd487782272 Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 13 Jan 2025 16:26:19 +0800 Subject: [PATCH 12/17] [#5721] improvement(mysql-catalog): add column not null limitation in unique index (#6183) ### What changes were proposed in this pull request? add column not null limitation in unique index ### Why are the changes needed? mysql will automatically change the null column in unique index to not null, so we add the limitation at creation Fix: #5721 ### Does this PR introduce _any_ user-facing change? yes, limitation for mysql unique index is more strict ### How was this patch tested? tests added --- .../mysql/operation/MysqlTableOperations.java | 30 +++++++++++++++++++ .../integration/test/CatalogMysqlIT.java | 21 +++++++++++++ .../operation/TestMysqlTableOperations.java | 4 +-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java index b8cc2f87233..36b4daebf9b 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -106,6 +106,7 @@ protected String generateCreateTableSql( } } + validateIndexes(indexes, columns); appendIndexesSql(indexes, sqlBuilder); sqlBuilder.append("\n)"); @@ -642,4 +643,33 @@ private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sq private static String quote(String name) { return BACK_QUOTE + name + BACK_QUOTE; } + + /** + * Verify the columns in the index. + * + * @param columns jdbc column + * @param indexes table indexes + */ + private static void validateIndexes(Index[] indexes, JdbcColumn[] columns) { + Map columnMap = + Arrays.stream(columns).collect(Collectors.toMap(JdbcColumn::name, c -> c)); + for (Index index : indexes) { + if (index.type() == Index.IndexType.UNIQUE_KEY) { + // the column in the unique index must be not null + for (String[] colNames : index.fieldNames()) { + JdbcColumn column = columnMap.get(colNames[0]); + Preconditions.checkArgument( + column != null, + "Column %s in the unique index %s does not exist in the table", + colNames[0], + index.name()); + Preconditions.checkArgument( + !column.nullable(), + "Column %s in the unique index %s must be a not null column", + colNames[0], + index.name()); + } + } + } + } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index a80da4795a0..9bd949b7b31 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -1037,6 +1037,27 @@ void testCreateTableIndex() { Assertions.assertEquals(2, table.index().length); Assertions.assertNotNull(table.index()[0].name()); Assertions.assertNotNull(table.index()[1].name()); + + Column notNullCol = Column.of("col_6", Types.LongType.get(), "id", true, false, null); + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> + tableCatalog.createTable( + tableIdent, + new Column[] {notNullCol}, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + new Index[] { + Indexes.of(Index.IndexType.UNIQUE_KEY, null, new String[][] {{"col_6"}}), + })); + Assertions.assertTrue( + exception + .getMessage() + .contains("Column col_6 in the unique index null must be a not null column")); } @Test diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java index 9eac348cd91..923e20fa0c0 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java @@ -64,7 +64,7 @@ public void testOperationTable() { .withName("col_1") .withType(VARCHAR) .withComment("test_comment") - .withNullable(true) + .withNullable(false) .build()); columns.add( JdbcColumn.builder() @@ -573,7 +573,7 @@ public void testCreateAndLoadTable() { JdbcColumn.builder() .withName("col_4") .withType(Types.DateType.get()) - .withNullable(true) + .withNullable(false) .withComment("date") .withDefaultValue(Column.DEFAULT_VALUE_NOT_SET) .build()); From 6138379dc316ef31e45c1594670f2e226c744d7a Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 13 Jan 2025 18:49:05 +0800 Subject: [PATCH 13/17] [#5100] improvement(docs): Add extra documents to clarify the engine type of MySQL catalog (#6209) ### What changes were proposed in this pull request? Add more details about the usage of engine type for MySQL catalog. ### Why are the changes needed? The value of engine type may be influenced by many factors like MySQL version, configurations and so on, we need to clarify it. Fix: #5100 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- docs/jdbc-mysql-catalog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index c761006a000..808e229a21d 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -186,6 +186,12 @@ Although MySQL itself does not support table properties, Gravitino offers table | `engine` | The engine used by the table. For example `MyISAM`, `MEMORY`, `CSV`, `ARCHIVE`, `BLACKHOLE`, `FEDERATED`, `ndbinfo`, `MRG_MYISAM`, `PERFORMANCE_SCHEMA`. | `InnoDB` | No | No | Yes | 0.4.0 | | `auto-increment-offset` | Used to specify the starting value of the auto-increment field. | (none) | No | No | Yes | 0.4.0 | + +:::note +Some MySQL storage engines, such as FEDERATED, are not enabled by default and require additional configuration to use. For example, to enable the FEDERATED engine, set federated=1 in the MySQL configuration file. Similarly, engines like ndbinfo, MRG_MYISAM, and PERFORMANCE_SCHEMA may also require specific prerequisites or configurations. For detailed instructions, +refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/federated-storage-engine.html). +::: + ### Table indexes - Supports PRIMARY_KEY and UNIQUE_KEY. From e4151e92e32864c603cb371559e19efdcae6262e Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:56:19 +0800 Subject: [PATCH 14/17] [#5192] [#5193] feat(flink): Support Catalog&Schema Operation DDL for paimon-catalog (#5818) ### What changes were proposed in this pull request? Support Catalog Operation DDL for paimon-catalog ### Why are the changes needed? Fix #5192 #5193 ### Does this PR introduce _any_ user-facing change? None ### How was this patch tested? org.apache.gravitino.flink.connector.paimon.TestPaimonPropertiesConverter org.apache.gravitino.flink.connector.integration.test.paimon.FlinkPaimonCatalogIT --- .../paimon/PaimonPropertiesUtils.java | 46 +++++--- flink-connector/flink/build.gradle.kts | 5 +- .../paimon/GravitinoPaimonCatalog.java | 48 ++++++++ .../paimon/GravitinoPaimonCatalogFactory.java | 80 +++++++++++++ .../GravitinoPaimonCatalogFactoryOptions.java | 26 ++++ .../paimon/PaimonPropertiesConverter.java | 80 +++++++++++++ .../store/GravitinoCatalogStore.java | 3 +- .../org.apache.flink.table.factories.Factory | 3 +- .../integration/test/FlinkCommonIT.java | 54 ++++++++- .../test/paimon/FlinkPaimonCatalogIT.java | 111 ++++++++++++++++++ .../paimon/TestPaimonPropertiesConverter.java | 101 ++++++++++++++++ 11 files changed, 536 insertions(+), 21 deletions(-) create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java index 0dcf24f3a67..7b1832fe56d 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java @@ -32,25 +32,41 @@ public class PaimonPropertiesUtils { // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will // change it to `catalogType` automatically and pass it to Paimon. public static final Map GRAVITINO_CONFIG_TO_PAIMON; + public static final Map PAIMON_CATALOG_CONFIG_TO_GRAVITINO; static { - Map map = new HashMap(); - map.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND); - map.put(PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER); - map.put(PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER); - map.put(PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD); - map.put(PaimonConstants.URI, PaimonConstants.URI); - map.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE); - map.put(PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME); + Map gravitinoConfigToPaimon = new HashMap<>(); + Map paimonCatalogConfigToGravitino = new HashMap<>(); + gravitinoConfigToPaimon.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD); + gravitinoConfigToPaimon.put(PaimonConstants.URI, PaimonConstants.URI); + gravitinoConfigToPaimon.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE); + gravitinoConfigToPaimon.put( + PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME); // S3 - map.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT); - map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY); - map.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY); + gravitinoConfigToPaimon.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT); + gravitinoConfigToPaimon.put( + S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY); + gravitinoConfigToPaimon.put( + S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY); // OSS - map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT); - map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY); - map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY); - GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(map); + gravitinoConfigToPaimon.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT); + gravitinoConfigToPaimon.put( + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY); + gravitinoConfigToPaimon.put( + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY); + GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(gravitinoConfigToPaimon); + gravitinoConfigToPaimon.forEach( + (key, value) -> { + paimonCatalogConfigToGravitino.put(value, key); + }); + PAIMON_CATALOG_CONFIG_TO_GRAVITINO = + Collections.unmodifiableMap(paimonCatalogConfigToGravitino); } /** diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 9e2a48c036c..f137a3eae1b 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -26,6 +26,7 @@ repositories { mavenCentral() } +var paimonVersion: String = libs.versions.paimon.get() val flinkVersion: String = libs.versions.flink.get() val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") @@ -38,14 +39,15 @@ val scalaVersion: String = "2.12" val artifactName = "${rootProject.name}-flink-${flinkMajorVersion}_$scalaVersion" dependencies { + implementation(project(":core")) implementation(project(":catalogs:catalog-common")) implementation(libs.guava) compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) - compileOnly("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") compileOnly("org.apache.flink:flink-table-common:$flinkVersion") compileOnly("org.apache.flink:flink-table-api-java:$flinkVersion") + compileOnly("org.apache.paimon:paimon-flink-1.18:$paimonVersion") compileOnly(libs.hive2.exec) { artifact { @@ -90,6 +92,7 @@ dependencies { testImplementation("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") testImplementation("org.apache.flink:flink-table-common:$flinkVersion") testImplementation("org.apache.flink:flink-table-api-java:$flinkVersion") + testImplementation("org.apache.paimon:paimon-flink-$flinkMajorVersion:$paimonVersion") testImplementation(libs.hive2.exec) { artifact { diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java new file mode 100644 index 00000000000..017ac6e7085 --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalog.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.paimon; + +import org.apache.flink.table.catalog.AbstractCatalog; +import org.apache.gravitino.flink.connector.PartitionConverter; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.apache.gravitino.flink.connector.catalog.BaseCatalog; + +/** + * The GravitinoPaimonCatalog class is an implementation of the BaseCatalog class that is used to + * proxy the PaimonCatalog class. + */ +public class GravitinoPaimonCatalog extends BaseCatalog { + + private final AbstractCatalog paimonCatalog; + + protected GravitinoPaimonCatalog( + String catalogName, + AbstractCatalog paimonCatalog, + PropertiesConverter propertiesConverter, + PartitionConverter partitionConverter) { + super(catalogName, paimonCatalog.getDefaultDatabase(), propertiesConverter, partitionConverter); + this.paimonCatalog = paimonCatalog; + } + + @Override + protected AbstractCatalog realCatalog() { + return paimonCatalog; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java new file mode 100644 index 00000000000..52489fc667f --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.paimon; + +import java.util.Collections; +import java.util.Set; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.table.catalog.Catalog; +import org.apache.gravitino.flink.connector.DefaultPartitionConverter; +import org.apache.gravitino.flink.connector.PartitionConverter; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; +import org.apache.paimon.flink.FlinkCatalog; +import org.apache.paimon.flink.FlinkCatalogFactory; + +/** + * Factory for creating instances of {@link GravitinoPaimonCatalog}. It will be created by SPI + * discovery in Flink. + */ +public class GravitinoPaimonCatalogFactory implements BaseCatalogFactory { + + @Override + public Catalog createCatalog(Context context) { + FlinkCatalog catalog = new FlinkCatalogFactory().createCatalog(context); + return new GravitinoPaimonCatalog( + context.getName(), catalog, propertiesConverter(), partitionConverter()); + } + + @Override + public String factoryIdentifier() { + return GravitinoPaimonCatalogFactoryOptions.IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + return Collections.emptySet(); + } + + @Override + public Set> optionalOptions() { + return Collections.emptySet(); + } + + @Override + public String gravitinoCatalogProvider() { + return "lakehouse-paimon"; + } + + @Override + public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { + return org.apache.gravitino.Catalog.Type.RELATIONAL; + } + + @Override + public PropertiesConverter propertiesConverter() { + return PaimonPropertiesConverter.INSTANCE; + } + + @Override + public PartitionConverter partitionConverter() { + return DefaultPartitionConverter.INSTANCE; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java new file mode 100644 index 00000000000..dd78f96d24b --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactoryOptions.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.paimon; + +public class GravitinoPaimonCatalogFactoryOptions { + + /** Identifier for the {@link GravitinoPaimonCatalog}. */ + public static final String IDENTIFIER = "gravitino-paimon"; +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java new file mode 100644 index 00000000000..58613bee37d --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/PaimonPropertiesConverter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.paimon; + +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants; +import org.apache.gravitino.catalog.lakehouse.paimon.PaimonPropertiesUtils; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.apache.paimon.catalog.FileSystemCatalogFactory; + +public class PaimonPropertiesConverter implements PropertiesConverter { + + public static final PaimonPropertiesConverter INSTANCE = new PaimonPropertiesConverter(); + + private PaimonPropertiesConverter() {} + + @Override + public Map toGravitinoCatalogProperties(Configuration flinkConf) { + Map gravitinoProperties = Maps.newHashMap(); + Map flinkConfMap = flinkConf.toMap(); + for (Map.Entry entry : flinkConfMap.entrySet()) { + String gravitinoKey = + PaimonPropertiesUtils.PAIMON_CATALOG_CONFIG_TO_GRAVITINO.get(entry.getKey()); + if (gravitinoKey != null) { + gravitinoProperties.put(gravitinoKey, entry.getValue()); + } else if (!entry.getKey().startsWith(FLINK_PROPERTY_PREFIX)) { + gravitinoProperties.put(FLINK_PROPERTY_PREFIX + entry.getKey(), entry.getValue()); + } else { + gravitinoProperties.put(entry.getKey(), entry.getValue()); + } + } + gravitinoProperties.put( + PaimonConstants.CATALOG_BACKEND, + flinkConfMap.getOrDefault(PaimonConstants.METASTORE, FileSystemCatalogFactory.IDENTIFIER)); + return gravitinoProperties; + } + + @Override + public Map toFlinkCatalogProperties(Map gravitinoProperties) { + Map all = new HashMap<>(); + gravitinoProperties.forEach( + (key, value) -> { + String flinkConfigKey = key; + if (key.startsWith(PropertiesConverter.FLINK_PROPERTY_PREFIX)) { + flinkConfigKey = key.substring(PropertiesConverter.FLINK_PROPERTY_PREFIX.length()); + } + all.put(flinkConfigKey, value); + }); + Map paimonCatalogProperties = + PaimonPropertiesUtils.toPaimonCatalogProperties(all); + paimonCatalogProperties.put( + PaimonConstants.METASTORE, + paimonCatalogProperties.getOrDefault( + PaimonConstants.CATALOG_BACKEND, FileSystemCatalogFactory.IDENTIFIER)); + paimonCatalogProperties.put( + CommonCatalogOptions.CATALOG_TYPE.key(), GravitinoPaimonCatalogFactoryOptions.IDENTIFIER); + return paimonCatalogProperties; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java index 92e778ce297..4c29b7fde3b 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java @@ -54,7 +54,8 @@ public GravitinoCatalogStore(GravitinoCatalogManager catalogManager) { public void storeCatalog(String catalogName, CatalogDescriptor descriptor) throws CatalogException { Configuration configuration = descriptor.getConfiguration(); - BaseCatalogFactory catalogFactory = getCatalogFactory(configuration.toMap()); + Map gravitino = configuration.toMap(); + BaseCatalogFactory catalogFactory = getCatalogFactory(gravitino); Map gravitinoProperties = catalogFactory.propertiesConverter().toGravitinoCatalogProperties(configuration); gravitinoCatalogManager.createCatalog( diff --git a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index c9d9c92b5ef..a535afb6dc2 100644 --- a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -18,4 +18,5 @@ # org.apache.gravitino.flink.connector.store.GravitinoCatalogStoreFactory -org.apache.gravitino.flink.connector.hive.GravitinoHiveCatalogFactory \ No newline at end of file +org.apache.gravitino.flink.connector.hive.GravitinoHiveCatalogFactory +org.apache.gravitino.flink.connector.paimon.GravitinoPaimonCatalogFactory \ No newline at end of file diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java index 2d022b4a8a4..5a363e4e51b 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -53,11 +54,24 @@ import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; public abstract class FlinkCommonIT extends FlinkEnvIT { protected abstract Catalog currentCatalog(); + protected boolean supportTableOperation() { + return true; + } + + protected boolean supportColumnOperation() { + return true; + } + + protected boolean supportSchemaOperationWithCommentAndOptions() { + return true; + } + @Test public void testCreateSchema() { doWithCatalog( @@ -76,7 +90,29 @@ public void testCreateSchema() { } @Test - public void testGetSchema() { + public void testGetSchemaWithoutCommentAndOption() { + doWithCatalog( + currentCatalog(), + catalog -> { + String schema = "test_get_schema"; + try { + TestUtils.assertTableResult( + sql("CREATE DATABASE IF NOT EXISTS %s", schema), ResultKind.SUCCESS); + TestUtils.assertTableResult(tableEnv.executeSql("USE " + schema), ResultKind.SUCCESS); + + catalog.asSchemas().schemaExists(schema); + Schema loadedSchema = catalog.asSchemas().loadSchema(schema); + Assertions.assertEquals(schema, loadedSchema.name()); + } finally { + catalog.asSchemas().dropSchema(schema, true); + Assertions.assertFalse(catalog.asSchemas().schemaExists(schema)); + } + }); + } + + @Test + @EnabledIf("supportSchemaOperationWithCommentAndOptions") + public void testGetSchemaWithCommentAndOptions() { doWithCatalog( currentCatalog(), catalog -> { @@ -114,7 +150,6 @@ public void testListSchema() { doWithCatalog( currentCatalog(), catalog -> { - Assertions.assertEquals(1, catalog.asSchemas().listSchemas().length); String schema = "test_list_schema"; String schema2 = "test_list_schema2"; String schema3 = "test_list_schema3"; @@ -135,6 +170,7 @@ public void testListSchema() { Row.of(schema3)); String[] schemas = catalog.asSchemas().listSchemas(); + Arrays.sort(schemas); Assertions.assertEquals(4, schemas.length); Assertions.assertEquals("default", schemas[0]); Assertions.assertEquals(schema, schemas[1]); @@ -150,7 +186,8 @@ public void testListSchema() { } @Test - public void testAlterSchema() { + @EnabledIf("supportSchemaOperationWithCommentAndOptions") + public void testAlterSchemaWithCommentAndOptions() { doWithCatalog( currentCatalog(), catalog -> { @@ -188,6 +225,7 @@ public void testAlterSchema() { } @Test + @EnabledIf("supportTableOperation") public void testCreateSimpleTable() { String databaseName = "test_create_no_partition_table_db"; String tableName = "test_create_no_partition_table"; @@ -236,6 +274,7 @@ public void testCreateSimpleTable() { } @Test + @EnabledIf("supportTableOperation") public void testListTables() { String newSchema = "test_list_table_catalog"; Column[] columns = new Column[] {Column.of("user_id", Types.IntegerType.get(), "USER_ID")}; @@ -268,6 +307,7 @@ public void testListTables() { } @Test + @EnabledIf("supportTableOperation") public void testDropTable() { String databaseName = "test_drop_table_db"; doWithSchema( @@ -289,6 +329,7 @@ public void testDropTable() { } @Test + @EnabledIf("supportTableOperation") public void testGetSimpleTable() { String databaseName = "test_get_simple_table"; Column[] columns = @@ -342,6 +383,7 @@ public void testGetSimpleTable() { } @Test + @EnabledIf("supportColumnOperation") public void testRenameColumn() { String databaseName = "test_rename_column_db"; String tableName = "test_rename_column"; @@ -377,6 +419,7 @@ public void testRenameColumn() { } @Test + @EnabledIf("supportColumnOperation") public void testAlterTableComment() { String databaseName = "test_alter_table_comment_database"; String tableName = "test_alter_table_comment"; @@ -436,6 +479,7 @@ public void testAlterTableComment() { } @Test + @EnabledIf("supportColumnOperation") public void testAlterTableAddColumn() { String databaseName = "test_alter_table_add_column_db"; String tableName = "test_alter_table_add_column"; @@ -471,6 +515,7 @@ public void testAlterTableAddColumn() { } @Test + @EnabledIf("supportColumnOperation") public void testAlterTableDropColumn() { String databaseName = "test_alter_table_drop_column_db"; String tableName = "test_alter_table_drop_column"; @@ -501,6 +546,7 @@ public void testAlterTableDropColumn() { } @Test + @EnabledIf("supportColumnOperation") public void testAlterColumnTypeAndChangeOrder() { String databaseName = "test_alter_table_alter_column_db"; String tableName = "test_alter_table_rename_column"; @@ -542,6 +588,7 @@ public void testAlterColumnTypeAndChangeOrder() { } @Test + @EnabledIf("supportTableOperation") public void testRenameTable() { String databaseName = "test_rename_table_db"; String tableName = "test_rename_table"; @@ -569,6 +616,7 @@ public void testRenameTable() { } @Test + @EnabledIf("supportTableOperation") public void testAlterTableProperties() { String databaseName = "test_alter_table_properties_db"; String tableName = "test_alter_table_properties"; diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java new file mode 100644 index 00000000000..10fab3567a3 --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.flink.connector.integration.test.paimon; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.nio.file.Path; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants; +import org.apache.gravitino.flink.connector.integration.test.FlinkCommonIT; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@Tag("gravitino-docker-test") +public class FlinkPaimonCatalogIT extends FlinkCommonIT { + + @TempDir private static Path warehouseDir; + + private static final String DEFAULT_PAIMON_CATALOG = + "test_flink_paimon_filesystem_schema_catalog"; + + private static org.apache.gravitino.Catalog catalog; + + @Override + protected boolean supportColumnOperation() { + return false; + } + + @Override + protected boolean supportTableOperation() { + return false; + } + + @Override + protected boolean supportSchemaOperationWithCommentAndOptions() { + return false; + } + + protected Catalog currentCatalog() { + return catalog; + } + + @BeforeAll + static void setup() { + initPaimonCatalog(); + } + + @AfterAll + static void stop() { + Preconditions.checkNotNull(metalake); + metalake.dropCatalog(DEFAULT_PAIMON_CATALOG, true); + } + + private static void initPaimonCatalog() { + Preconditions.checkNotNull(metalake); + catalog = + metalake.createCatalog( + DEFAULT_PAIMON_CATALOG, + org.apache.gravitino.Catalog.Type.RELATIONAL, + "lakehouse-paimon", + null, + ImmutableMap.of( + PaimonConstants.CATALOG_BACKEND, + "filesystem", + "warehouse", + warehouseDir.toString())); + } + + @Test + public void testCreateGravitinoPaimonCatalogUsingSQL() { + tableEnv.useCatalog(DEFAULT_CATALOG); + int numCatalogs = tableEnv.listCatalogs().length; + String catalogName = "gravitino_hive_sql"; + String warehouse = warehouseDir.toString(); + tableEnv.executeSql( + String.format( + "create catalog %s with (" + + "'type'='gravitino-paimon', " + + "'warehouse'='%s'," + + "'catalog.backend'='filesystem'" + + ")", + catalogName, warehouse)); + String[] catalogs = tableEnv.listCatalogs(); + Assertions.assertEquals(numCatalogs + 1, catalogs.length, "Should create a new catalog"); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + org.apache.gravitino.Catalog gravitinoCatalog = metalake.loadCatalog(catalogName); + Map properties = gravitinoCatalog.properties(); + Assertions.assertEquals(warehouse, properties.get("warehouse")); + } +} diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java new file mode 100644 index 00000000000..4496d94c0a4 --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/paimon/TestPaimonPropertiesConverter.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.flink.connector.paimon; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.flink.configuration.Configuration; +import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConstants; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Test for {@link PaimonPropertiesConverter} */ +public class TestPaimonPropertiesConverter { + + private static final PaimonPropertiesConverter CONVERTER = PaimonPropertiesConverter.INSTANCE; + + private static final String localWarehouse = "file:///tmp/paimon_warehouse"; + + @Test + public void testToPaimonFileSystemCatalog() { + Map catalogProperties = ImmutableMap.of("warehouse", localWarehouse); + Map flinkCatalogProperties = + CONVERTER.toFlinkCatalogProperties(catalogProperties); + Assertions.assertEquals( + GravitinoPaimonCatalogFactoryOptions.IDENTIFIER, flinkCatalogProperties.get("type")); + Assertions.assertEquals(localWarehouse, flinkCatalogProperties.get("warehouse")); + } + + @Test + public void testToPaimonJdbcCatalog() { + String testUser = "testUser"; + String testPassword = "testPassword"; + String testUri = "testUri"; + Map catalogProperties = + ImmutableMap.of( + PaimonConstants.WAREHOUSE, + localWarehouse, + PaimonConstants.CATALOG_BACKEND, + "jdbc", + PaimonConstants.GRAVITINO_JDBC_USER, + testUser, + PaimonConstants.GRAVITINO_JDBC_PASSWORD, + testPassword, + PropertiesConverter.FLINK_PROPERTY_PREFIX + PaimonConstants.URI, + testUri); + Map flinkCatalogProperties = + CONVERTER.toFlinkCatalogProperties(catalogProperties); + Assertions.assertEquals( + GravitinoPaimonCatalogFactoryOptions.IDENTIFIER, flinkCatalogProperties.get("type")); + Assertions.assertEquals(localWarehouse, flinkCatalogProperties.get(PaimonConstants.WAREHOUSE)); + Assertions.assertEquals(testUser, flinkCatalogProperties.get(PaimonConstants.PAIMON_JDBC_USER)); + Assertions.assertEquals( + testPassword, flinkCatalogProperties.get(PaimonConstants.PAIMON_JDBC_PASSWORD)); + Assertions.assertEquals("jdbc", flinkCatalogProperties.get(PaimonConstants.METASTORE)); + Assertions.assertEquals(testUri, flinkCatalogProperties.get(PaimonConstants.URI)); + } + + @Test + public void testToGravitinoCatalogProperties() { + String testUser = "testUser"; + String testPassword = "testPassword"; + String testUri = "testUri"; + String testBackend = "jdbc"; + Configuration configuration = + Configuration.fromMap( + ImmutableMap.of( + PaimonConstants.WAREHOUSE, + localWarehouse, + PaimonConstants.METASTORE, + testBackend, + PaimonConstants.PAIMON_JDBC_USER, + testUser, + PaimonConstants.PAIMON_JDBC_PASSWORD, + testPassword, + PaimonConstants.URI, + testUri)); + Map properties = CONVERTER.toGravitinoCatalogProperties(configuration); + Assertions.assertEquals(localWarehouse, properties.get(PaimonConstants.WAREHOUSE)); + Assertions.assertEquals(testUser, properties.get(PaimonConstants.GRAVITINO_JDBC_USER)); + Assertions.assertEquals(testPassword, properties.get(PaimonConstants.GRAVITINO_JDBC_PASSWORD)); + Assertions.assertEquals(testUri, properties.get(PaimonConstants.URI)); + Assertions.assertEquals(testBackend, properties.get(PaimonConstants.CATALOG_BACKEND)); + } +} From 08f47ad4551913e7c05d2c5b08572cc342b78a5f Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Tue, 14 Jan 2025 10:34:27 +1100 Subject: [PATCH 15/17] [#6139] Refactor metalake command in Gravitino CLI (#6140) ### What changes were proposed in this pull request? The Gravitino command line class is a little large and could be broken up. ### Why are the changes needed? For readability and maintainability. Fix: #6139 ### Does this PR introduce _any_ user-facing change? None. ### How was this patch tested? Tested locally. --------- Co-authored-by: Shaofeng Shi --- .../gravitino/cli/GravitinoCommandLine.java | 87 +------- .../gravitino/cli/MetalakeCommandHandler.java | 201 ++++++++++++++++++ 2 files changed, 202 insertions(+), 86 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 21d3ed176cb..cb8663ef379 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -137,7 +137,7 @@ private void executeCommand() { } else if (entity.equals(CommandEntities.CATALOG)) { new CatalogCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.METALAKE)) { - handleMetalakeCommand(); + new MetalakeCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TOPIC)) { new TopicCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.FILESET)) { @@ -155,91 +155,6 @@ private void executeCommand() { } } - /** - * Handles the command execution for Metalakes based on command type and the command line options. - */ - private void handleMetalakeCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); - - Command.setAuthenticationMode(auth, userName); - - if (CommandActions.LIST.equals(command)) { - newListMetalakes(url, ignore, outputFormat).validate().handle(); - return; - } - - String metalake = name.getMetalakeName(); - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newMetalakeAudit(url, ignore, metalake).validate().handle(); - } else { - newMetalakeDetails(url, ignore, outputFormat, metalake).validate().handle(); - } - break; - - case CommandActions.CREATE: - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newCreateMetalake(url, ignore, metalake, comment).validate().handle(); - break; - - case CommandActions.DELETE: - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteMetalake(url, ignore, force, metalake).validate().handle(); - break; - - case CommandActions.SET: - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - String value = line.getOptionValue(GravitinoOptions.VALUE); - newSetMetalakeProperty(url, ignore, metalake, property, value).validate().handle(); - break; - - case CommandActions.REMOVE: - property = line.getOptionValue(GravitinoOptions.PROPERTY); - newRemoveMetalakeProperty(url, ignore, metalake, property).validate().handle(); - break; - - case CommandActions.PROPERTIES: - newListMetalakeProperties(url, ignore, metalake).validate().handle(); - break; - - case CommandActions.UPDATE: - if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { - System.err.println(ErrorMessages.INVALID_ENABLE_DISABLE); - Main.exit(-1); - } - if (line.hasOption(GravitinoOptions.ENABLE)) { - boolean enableAllCatalogs = line.hasOption(GravitinoOptions.ALL); - newMetalakeEnable(url, ignore, metalake, enableAllCatalogs).validate().handle(); - } - if (line.hasOption(GravitinoOptions.DISABLE)) { - newMetalakeDisable(url, ignore, metalake).validate().handle(); - } - - if (line.hasOption(GravitinoOptions.COMMENT)) { - comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateMetalakeComment(url, ignore, metalake, comment).validate().handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - force = line.hasOption(GravitinoOptions.FORCE); - newUpdateMetalakeName(url, ignore, force, metalake, newName).validate().handle(); - } - - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); - Main.exit(-1); - break; - } - } - /** Handles the command execution for Tags based on command type and the command line options. */ protected void handleTagCommand() { String url = getUrl(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java new file mode 100644 index 00000000000..993116f19f5 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** + * Handles the command execution for Metalakes based on command type and the command line options. + */ +public class MetalakeCommandHandler extends CommandHandler { + + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private String metalake; + + /** + * Constructs a MetalakeCommandHandler instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public MetalakeCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + this.url = getUrl(line); + } + + /** Handles the command execution logic based on the provided command. */ + public void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + FullName name = new FullName(line); + Command.setAuthenticationMode(getAuth(line), userName); + + if (CommandActions.LIST.equals(command)) { + handleListCommand(); + return; + } + + metalake = name.getMetalakeName(); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); + gravitinoCommandLine.newListMetalakes(url, ignore, outputFormat).validate().handle(); + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + gravitinoCommandLine.newMetalakeAudit(url, ignore, metalake).validate().handle(); + } else { + String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); + gravitinoCommandLine + .newMetalakeDetails(url, ignore, outputFormat, metalake) + .validate() + .handle(); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine.newCreateMetalake(url, ignore, metalake, comment).validate().handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine.newDeleteMetalake(url, ignore, force, metalake).validate().handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + gravitinoCommandLine + .newSetMetalakeProperty(url, ignore, metalake, property, value) + .validate() + .handle(); + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + gravitinoCommandLine + .newRemoveMetalakeProperty(url, ignore, metalake, property) + .validate() + .handle(); + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine.newListMetalakeProperties(url, ignore, metalake).validate().handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { + System.err.println(ErrorMessages.INVALID_ENABLE_DISABLE); + Main.exit(-1); + } + if (line.hasOption(GravitinoOptions.ENABLE)) { + boolean enableAllCatalogs = line.hasOption(GravitinoOptions.ALL); + gravitinoCommandLine + .newMetalakeEnable(url, ignore, metalake, enableAllCatalogs) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DISABLE)) { + gravitinoCommandLine.newMetalakeDisable(url, ignore, metalake).validate().handle(); + } + + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateMetalakeComment(url, ignore, metalake, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newUpdateMetalakeName(url, ignore, force, metalake, newName) + .validate() + .handle(); + } + } +} From 546a9771a44f423e6b0b16a550205b7ea689beb3 Mon Sep 17 00:00:00 2001 From: TungYuChiang <75083792+TungYuChiang@users.noreply.github.com> Date: Tue, 14 Jan 2025 07:36:53 +0800 Subject: [PATCH 16/17] =?UTF-8?q?[#6147]=20improve(CLI):=20Refactor=20file?= =?UTF-8?q?set=20commands=20in=20Gavitino=20CLI=C2=A0=20(#6191)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? Refactor fileset commands in cli client  ### Why are the changes needed? Fix: #6147 ### Does this PR introduce _any_ user-facing change? None ### How was this patch tested? Tested locally --- .../gravitino/cli/FilesetCommandHandler.java | 210 ++++++++++++++++++ .../gravitino/cli/GravitinoCommandLine.java | 106 +-------- 2 files changed, 211 insertions(+), 105 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java new file mode 100644 index 00000000000..33fc1fe9ee7 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** + * Handles the command execution for Filesets based on command type and the command line options. + */ +public class FilesetCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String catalog; + private final String schema; + private String fileset; + + /** + * Constructs a {@link FilesetCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public FilesetCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = gravitinoCommandLine.getUrl(); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.catalog = name.getCatalogName(); + this.schema = name.getSchemaName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(gravitinoCommandLine.getAuth(), userName); + + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); + handleListCommand(); + return; + } + + this.fileset = name.getFilesetName(); + if (fileset == null) missingEntities.add(CommandEntities.FILESET); + checkEntities(missingEntities); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine + .newFilesetDetails(url, ignore, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + String[] properties = line.getOptionValues(CommandActions.PROPERTIES); + Map propertyMap = new Properties().parse(properties); + gravitinoCommandLine + .newCreateFileset(url, ignore, metalake, catalog, schema, fileset, comment, propertyMap) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newDeleteFileset(url, ignore, force, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + gravitinoCommandLine + .newSetFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property, value) + .validate() + .handle(); + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + gravitinoCommandLine + .newRemoveFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property) + .validate() + .handle(); + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine + .newListFilesetProperties(url, ignore, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + gravitinoCommandLine + .newListFilesets(url, ignore, metalake, catalog, schema) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateFilesetComment(url, ignore, metalake, catalog, schema, fileset, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateFilesetName(url, ignore, metalake, catalog, schema, fileset, newName) + .validate() + .handle(); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index cb8663ef379..dd98ebf50d3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -20,7 +20,6 @@ package org.apache.gravitino.cli; import com.google.common.base.Joiner; -import com.google.common.collect.Lists; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -28,7 +27,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; @@ -141,7 +139,7 @@ private void executeCommand() { } else if (entity.equals(CommandEntities.TOPIC)) { new TopicCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.FILESET)) { - handleFilesetCommand(); + new FilesetCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.USER)) { new UserCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.GROUP)) { @@ -273,108 +271,6 @@ private void handleHelpCommand() { } } - /** - * Handles the command execution for filesets based on command type and the command line options. - */ - private void handleFilesetCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - - Command.setAuthenticationMode(auth, userName); - - List missingEntities = Lists.newArrayList(); - if (catalog == null) missingEntities.add(CommandEntities.CATALOG); - if (schema == null) missingEntities.add(CommandEntities.SCHEMA); - - // Handle CommandActions.LIST action separately as it doesn't require the `fileset` - if (CommandActions.LIST.equals(command)) { - checkEntities(missingEntities); - newListFilesets(url, ignore, metalake, catalog, schema).validate().handle(); - return; - } - - String fileset = name.getFilesetName(); - if (fileset == null) missingEntities.add(CommandEntities.FILESET); - checkEntities(missingEntities); - - switch (command) { - case CommandActions.DETAILS: - newFilesetDetails(url, ignore, metalake, catalog, schema, fileset).validate().handle(); - break; - - case CommandActions.CREATE: - { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - String[] properties = line.getOptionValues(CommandActions.PROPERTIES); - Map propertyMap = new Properties().parse(properties); - newCreateFileset(url, ignore, metalake, catalog, schema, fileset, comment, propertyMap) - .validate() - .handle(); - break; - } - - case CommandActions.DELETE: - { - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteFileset(url, ignore, force, metalake, catalog, schema, fileset) - .validate() - .handle(); - break; - } - - case CommandActions.SET: - { - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - String value = line.getOptionValue(GravitinoOptions.VALUE); - newSetFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property, value) - .validate() - .handle(); - break; - } - - case CommandActions.REMOVE: - { - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - newRemoveFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property) - .validate() - .handle(); - break; - } - - case CommandActions.PROPERTIES: - newListFilesetProperties(url, ignore, metalake, catalog, schema, fileset) - .validate() - .handle(); - break; - - case CommandActions.UPDATE: - { - if (line.hasOption(GravitinoOptions.COMMENT)) { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateFilesetComment(url, ignore, metalake, catalog, schema, fileset, comment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateFilesetName(url, ignore, metalake, catalog, schema, fileset, newName) - .validate() - .handle(); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - break; - } - } - /** * Retrieves the Gravitinno URL from the command line options or the GRAVITINO_URL environment * variable or the Gravitio config file. From ea790d7e03aa9505734dd337f6e340c04a1d638a Mon Sep 17 00:00:00 2001 From: TengYao Chi Date: Tue, 14 Jan 2025 10:35:03 +0800 Subject: [PATCH 17/17] [#6152] refactor: Refactor tag commands in Gravitino CLI (#6192) ### What changes were proposed in this pull request? Reduce complexity in `GravitinoCommandLine` ### Why are the changes needed? For readability and maintainability. Fix: #6152 ### Does this PR introduce _any_ user-facing change? (Please list the user-facing changes introduced by your change, including None. ### How was this patch tested? Tested locally. --------- Co-authored-by: Justin Mclean --- .../gravitino/cli/GravitinoCommandLine.java | 104 +-------- .../gravitino/cli/TagCommandHandler.java | 207 ++++++++++++++++++ 2 files changed, 208 insertions(+), 103 deletions(-) create mode 100644 clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index dd98ebf50d3..11737206067 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -25,12 +25,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; -import org.apache.gravitino.cli.commands.Command; /* Gravitino Command line */ public class GravitinoCommandLine extends TestableCommandLine { @@ -145,7 +143,7 @@ private void executeCommand() { } else if (entity.equals(CommandEntities.GROUP)) { new GroupCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TAG)) { - handleTagCommand(); + new TagCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.ROLE)) { new RoleCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.MODEL)) { @@ -153,106 +151,6 @@ private void executeCommand() { } } - /** Handles the command execution for Tags based on command type and the command line options. */ - protected void handleTagCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - - Command.setAuthenticationMode(auth, userName); - - String[] tags = line.getOptionValues(GravitinoOptions.TAG); - - if (tags != null) { - tags = Arrays.stream(tags).distinct().toArray(String[]::new); - } - - switch (command) { - case CommandActions.DETAILS: - newTagDetails(url, ignore, metalake, getOneTag(tags)).validate().handle(); - break; - - case CommandActions.LIST: - if (!name.hasCatalogName()) { - newListTags(url, ignore, metalake).validate().handle(); - } else { - newListEntityTags(url, ignore, metalake, name).validate().handle(); - } - break; - - case CommandActions.CREATE: - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newCreateTags(url, ignore, metalake, tags, comment).validate().handle(); - break; - - case CommandActions.DELETE: - boolean forceDelete = line.hasOption(GravitinoOptions.FORCE); - newDeleteTag(url, ignore, forceDelete, metalake, tags).validate().handle(); - break; - - case CommandActions.SET: - String propertySet = line.getOptionValue(GravitinoOptions.PROPERTY); - String valueSet = line.getOptionValue(GravitinoOptions.VALUE); - if (propertySet == null && valueSet == null) { - newTagEntity(url, ignore, metalake, name, tags).validate().handle(); - } else { - newSetTagProperty(url, ignore, metalake, getOneTag(tags), propertySet, valueSet) - .validate() - .handle(); - } - break; - - case CommandActions.REMOVE: - boolean isTag = line.hasOption(GravitinoOptions.TAG); - if (!isTag) { - boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); - newRemoveAllTags(url, ignore, metalake, name, forceRemove).validate().handle(); - } else { - String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); - if (propertyRemove != null) { - newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove) - .validate() - .handle(); - } else { - newUntagEntity(url, ignore, metalake, name, tags).validate().handle(); - } - } - break; - - case CommandActions.PROPERTIES: - newListTagProperties(url, ignore, metalake, getOneTag(tags)).validate().handle(); - break; - - case CommandActions.UPDATE: - if (line.hasOption(GravitinoOptions.COMMENT)) { - String updateComment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateTagComment(url, ignore, metalake, getOneTag(tags), updateComment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateTagName(url, ignore, metalake, getOneTag(tags), newName).validate().handle(); - } - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - break; - } - } - - private String getOneTag(String[] tags) { - if (tags == null || tags.length > 1) { - System.err.println(ErrorMessages.MULTIPLE_TAG_COMMAND_ERROR); - Main.exit(-1); - } - return tags[0]; - } - private void handleHelpCommand() { String helpFile = entity.toLowerCase() + "_help.txt"; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java new file mode 100644 index 00000000000..e274c271f9c --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli; + +import java.util.Arrays; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +public class TagCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private String[] tags; + private String metalake; + + public TagCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + this.url = getUrl(line); + this.tags = line.getOptionValues(GravitinoOptions.TAG); + + if (tags != null) { + tags = Arrays.stream(tags).distinct().toArray(String[]::new); + } + } + + @Override + public void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + FullName name = new FullName(line); + Command.setAuthenticationMode(getAuth(line), userName); + + metalake = name.getMetalakeName(); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.LIST: + handleListCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + FullName name = new FullName(line); + if (!name.hasCatalogName()) { + gravitinoCommandLine.newListTags(url, ignore, metalake).validate().handle(); + } else { + gravitinoCommandLine.newListEntityTags(url, ignore, metalake, name).validate().handle(); + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine.newTagDetails(url, ignore, metalake, getOneTag(tags)).validate().handle(); + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine.newCreateTags(url, ignore, metalake, tags, comment).validate().handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean forceDelete = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine.newDeleteTag(url, ignore, forceDelete, metalake, tags).validate().handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + if (property == null && value == null) { + gravitinoCommandLine + .newTagEntity(url, ignore, metalake, new FullName(line), tags) + .validate() + .handle(); + } else { + gravitinoCommandLine + .newSetTagProperty(url, ignore, metalake, getOneTag(tags), property, value) + .validate() + .handle(); + } + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + boolean isTag = line.hasOption(GravitinoOptions.TAG); + FullName name = new FullName(line); + if (!isTag) { + boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newRemoveAllTags(url, ignore, metalake, name, forceRemove) + .validate() + .handle(); + } else { + String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); + if (propertyRemove != null) { + gravitinoCommandLine + .newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove) + .validate() + .handle(); + } else { + gravitinoCommandLine.newUntagEntity(url, ignore, metalake, name, tags).validate().handle(); + } + } + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine + .newListTagProperties(url, ignore, metalake, getOneTag(tags)) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + + if (line.hasOption(GravitinoOptions.COMMENT)) { + String updateComment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateTagComment(url, ignore, metalake, getOneTag(tags), updateComment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateTagName(url, ignore, metalake, getOneTag(tags), newName) + .validate() + .handle(); + } + } + + private String getOneTag(String[] tags) { + if (tags == null || tags.length > 1) { + System.err.println(ErrorMessages.MULTIPLE_TAG_COMMAND_ERROR); + Main.exit(-1); + } + return tags[0]; + } +}