[DAE-92] Add method create_external_table (#42)

* Add new method create_external_table

* Add example for create_external_table

* Update methods list

* Remove uncessary assertion from test

* Update docstring

* Add example in docs

Author: Lucas Mendes Mota da Fonseca

docs/source/getstarted.md

@@ -22,9 +22,11 @@ Click on the following links to open the [examples](https://github.com/quintoand
 
 **[#2 Create a table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/create_table.py)**
 
-**[#3 Add columns to a table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/add_columns_to_table.py)**
+**[#3 Create an external table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/create_external_table.py)**
 
-**[#4 Add partitions to a table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/add_partitions.py)**
+**[#4 Add columns to a table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/add_columns_to_table.py)**
+
+**[#5 Add partitions to a table](https://github.com/quintoandar/hive-metastore-client/blob/main/examples/add_partitions.py)**
 
 ## Available methods
 
@@ -38,4 +40,5 @@ the [`HiveMetastoreClient`](https://github.com/quintoandar/hive-metastore-client
 - [`add_columns_to_table`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.add_columns_to_table)
 - [`drop_columns_from_table`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.drop_columns_from_table)
 - [`add_partitions_if_not_exists`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.add_partitions_if_not_exists)
-- [`create_database_if_not_exists`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.create_database_if_not_exists)
+- [`create_database_if_not_exists`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.create_database_if_not_exists)
+- [`create_external_table`](https://hive-metastore-client.readthedocs.io/en/latest/hive_metastore_client.html#hive_metastore_client.hive_metastore_client.HiveMetastoreClient.create_external_table)

examples/create_external_table.py

@@ -0,0 +1,65 @@
+"""
+    The thrift Table object requires others objects as arguments.
+    Use the builders for creating each of them.
+    Some arguments are optional when creating a thrift object.
+    Check each Builder constructor for more information.
+
+    Due to a bug in Hive Metastore server we need to enforce the parameter
+     EXTERNAL=TRUE when creating an external table. You can either use the
+     method `create_external_table` with the table object or declare the two
+     table parameters before calling the method create_table.
+"""
+
+from hive_metastore_client import HiveMetastoreClient
+from hive_metastore_client.builders import (
+    ColumnBuilder,
+    SerDeInfoBuilder,
+    StorageDescriptorBuilder,
+    TableBuilder,
+)
+
+HIVE_HOST = "<ADD_HIVE_HOST_HERE>"
+HIVE_PORT = 9083
+
+# You must create a list with the columns
+columns = [
+    ColumnBuilder("id", "string", "col comment").build(),
+    ColumnBuilder("client_name", "string").build(),
+    ColumnBuilder("amount", "string").build(),
+    ColumnBuilder("year", "string").build(),
+    ColumnBuilder("month", "string").build(),
+    ColumnBuilder("day", "string").build(),
+]
+
+# If you table has partitions create a list with the partition columns
+# This list is similar to the columns list, and the year, month and day
+# columns are the same.
+partition_keys = [
+    ColumnBuilder("year", "string").build(),
+    ColumnBuilder("month", "string").build(),
+    ColumnBuilder("day", "string").build(),
+]
+
+serde_info = SerDeInfoBuilder(
+    serialization_lib="org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
+).build()
+
+storage_descriptor = StorageDescriptorBuilder(
+    columns=columns,
+    location="s3a://path/to/file",
+    input_format="org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+    output_format="org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
+    serde_info=serde_info,
+).build()
+
+table = TableBuilder(
+    table_name="orders",
+    db_name="store",
+    owner="owner name",
+    storage_descriptor=storage_descriptor,
+    partition_keys=partition_keys,
+).build()
+
+with HiveMetastoreClient(HIVE_HOST, HIVE_PORT) as hive_metastore_client:
+    # Creating new table from thrift table object
+    hive_metastore_client.create_external_table(table)

examples/create_table.py

@@ -3,6 +3,9 @@
     Use the builders for creating each of them.
     Some arguments are optional when creating a thrift object.
     Check each Builder constructor for more information.
+
+    Note: if you want to create an external table refer to the
+     example create_external_table.py.
 """
 
 from hive_metastore_client import HiveMetastoreClient
@@ -41,9 +44,9 @@
 
 storage_descriptor = StorageDescriptorBuilder(
     columns=columns,
-    location="path/to/file",
-    input_format="org.apache.hadoop.mapred.TextInputFormat",
-    output_format="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+    location="s3a://path/to/file",
+    input_format="org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+    output_format="org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
     serde_info=serde_info,
 ).build()
 

hive_metastore_client/hive_metastore_client.py

@@ -14,6 +14,7 @@
     FieldSchema,
     Database,
     AlreadyExistsException,
+    Table,
 )
 
 
@@ -159,6 +160,22 @@ def create_database_if_not_exists(self, database: Database) -> None:
         except AlreadyExistsException:
             pass
 
+    def create_external_table(self, table: Table) -> None:
+        """
+        Creates an external table in Hive Metastore.
+
+        When a table is created with tableType default (None) or equal to
+         EXTERNAL_TABLE there is an error in the server that creates the table
+         as a MANAGED_TABLE.
+        This method enforces the parameter EXTERNAL=TRUE so the table is
+         created correctly.
+
+        :param table: the table object
+        """
+        table.parameters = {"EXTERNAL": "TRUE"}
+        table.tableType = "EXTERNAL_TABLE"
+        self.create_table(table)
+
     @staticmethod
     def _format_partitions_location(
         partition_list: List[Partition],

tests/unit/hive_metastore_client/test_hive_metastore_client.py

@@ -1,10 +1,12 @@
+from copy import copy
 from unittest import mock
 from unittest.mock import Mock, ANY
 
 import pytest
 from pytest import raises
 
 from hive_metastore_client import HiveMetastoreClient
+from hive_metastore_client.builders import TableBuilder
 from thrift_files.libraries.thrift_hive_metastore_client.ThriftHiveMetastore import (
     Client as ThriftClient,
 )
@@ -295,3 +297,23 @@ def test_create_database_if_not_exists_with_existent_database(
 
         # assert
         mocked_create_database.assert_called_once_with(mocked_database_obj)
+
+    @mock.patch.object(HiveMetastoreClient, "create_table")
+    def test_create_external_table(self, mocked_create_table, hive_metastore_client):
+        # arrange
+        table = TableBuilder(
+            table_name="table_name",
+            db_name="database_name",
+            owner="owner",
+            storage_descriptor=Mock(),
+            partition_keys=[],
+        ).build()
+        updated_table = copy(table)
+        updated_table.parameters = {"EXTERNAL": "TRUE"}
+        updated_table.tableType = "EXTERNAL_TABLE"
+
+        # act
+        hive_metastore_client.create_external_table(table)
+
+        # assert
+        mocked_create_table.assert_called_once_with(updated_table)