Merge pull request #19 from dangtrungtin/master

Improve peformances put/get rows using pandas.DataFrame

Author: knonomura

Makefile

@@ -9,7 +9,8 @@ CPPFLAGS = -fPIC -std=c++0x -g -O2
 INCLUDES = -Iinclude -Isrc
 
 INCLUDES_PYTHON = $(INCLUDES)	\
-				-I/usr/include/python3.6
+				-I/usr/include/python3.6		\
+				-I$(HOME)/.pyenv/versions/3.6.4/lib/python3.6/site-packages/numpy/core/include
 
 PROGRAM = _griddb_python.so
 EXTRA = griddb_python.py griddb_python.pyc
@@ -24,6 +25,7 @@ SOURCES = 	  src/TimeSeriesProperties.cpp \
 		  src/Query.cpp				\
 		  src/QueryAnalysisEntry.cpp			\
 		  src/RowKeyPredicate.cpp	\
+		  src/RowList.cpp	\
 		  src/RowSet.cpp			\
 		  src/TimestampUtils.cpp			\
 		  src/Field.cpp \

README.md

@@ -49,6 +49,11 @@ Set CPATH and LIBRARY_PATH.
 
     export LIBRARY_PATH=$LIBRARY_PATH:<C client library file directory path>
 
+Install Pandas and Numpy as below:
+
+    $ python -m pip install numpy
+    $ python -m pip install pandas
+
 ### Build and Run 
 
     1. Execute the command on project directory.

sample/FetchRowsWithDataFrame.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python
+
+import griddb_python as griddb
+import sys
+import pandas
+
+factory = griddb.StoreFactory.get_instance()
+
+argv = sys.argv
+
+blob = bytearray([65, 66, 67, 68, 69, 70, 71, 72, 73, 74])
+containerName = "SamplePython_FetchRows"
+update = False
+
+try:
+    # Get GridStore object
+    gridstore = factory.get_store(host=argv[1], port=int(argv[2]), cluster_name=argv[3], username=argv[4], password=argv[5])
+
+    # Create Collection
+    conInfo = griddb.ContainerInfo(containerName,
+                    [["name", griddb.Type.STRING],
+                    ["status", griddb.Type.BOOL],
+                    ["count", griddb.Type.LONG],
+                    ["lob", griddb.Type.BLOB]],
+                    griddb.ContainerType.COLLECTION, True)
+    col = gridstore.put_container(conInfo)
+    print("Create Collection name=", containerName)
+
+    # Put rows
+    rows = pandas.DataFrame([["name01", True, 1, blob], ["name02", False, 2, blob]])
+    col.put_rows(rows)
+    print("Put rows with DataFrame")
+
+    # Fetch rows
+    query = col.query("select *")
+    rs = query.fetch(update)
+    print("Fetch rows with DataFrame")
+    result = rs.fetch_rows()
+    print(result)
+    print("Success!")
+
+except griddb.GSException as e:
+    for i in range(e.get_error_stack_size()):
+        print("[", i, "]")
+        print(e.get_error_code(i))
+        print(e.get_location(i))
+        print(e.get_message(i))

sample/PutRowsWithDataFrame.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+
+import griddb_python as griddb
+import sys
+import pandas
+
+factory = griddb.StoreFactory.get_instance()
+
+argv = sys.argv
+
+blob = bytearray([65, 66, 67, 68, 69, 70, 71, 72, 73, 74])
+update = False
+containerName = "SamplePython_PutRows"
+
+try:
+    # Get GridStore object
+    gridstore = factory.get_store(host=argv[1], port=int(argv[2]), cluster_name=argv[3], username=argv[4], password=argv[5])
+
+    # Create Collection
+    conInfo = griddb.ContainerInfo(containerName,
+                    [["name", griddb.Type.STRING],
+                    ["status", griddb.Type.BOOL],
+                    ["count", griddb.Type.LONG],
+                    ["lob", griddb.Type.BLOB]],
+                    griddb.ContainerType.COLLECTION, True)
+    col = gridstore.put_container(conInfo)
+    print("Create Collection name=", containerName)
+
+    # Put rows
+    rows = pandas.DataFrame([["name01", False, 1, blob], ["name02", False, 1, blob]])
+    col.put_rows(rows)
+    print("Put rows with DataFrame")
+    print("Success!")
+
+except griddb.GSException as e:
+    for i in range(e.get_error_stack_size()):
+        print("[", i, "]")
+        print(e.get_error_code(i))
+        print(e.get_location(i))
+        print(e.get_message(i))

src/Container.cpp

@@ -61,7 +61,7 @@ namespace griddb {
             freeMemoryContainer();
             throw GSException(mContainer, "Memory allocation error");
         }
-        
+
         mContainerInfo->timeSeriesProperties = NULL;
         mContainerInfo->triggerInfoList = NULL;
         mContainerInfo->dataAffinity = NULL;
@@ -74,8 +74,6 @@ namespace griddb {
     }
 
     Container::~Container() {
-        
-
     // allRelated = FALSE, since all row object is managed by Row class
         close(GS_FALSE);
     }
@@ -409,4 +407,11 @@ namespace griddb {
     int Container::getColumnCount(){
         return mContainerInfo->columnCount;
     }
+
+    /**
+     * @brief Put rows with input is numpy data
+     */
+    void Container::put_rows(GSRow** listRow, int rowCount) {
+        this->multi_put(listRow, rowCount);
+    }
 }

src/Container.h

@@ -58,6 +58,7 @@ class Container {
         GSType* getGSTypeList();
         int getColumnCount();
         GSRow* getGSRowPtr();
+        void put_rows(GSRow** listRow, int rowCount);
 
     private:
         Container(GSContainer *container, GSContainerInfo* containerInfo);

src/Query.cpp

@@ -0,0 +1,69 @@
+/*
+    Copyright (c) 2017 TOSHIBA Digital Solutions Corporation.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+#include "RowList.h"
+
+namespace griddb {
+
+    RowList::RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType *typelist,
+            int columnCount, bool timetampFloat) :
+            mRowSet(gsRowSet), mRow(gsRow), mTypelist(typelist),
+            mColumnCount(columnCount), mTimetampFloat(timetampFloat) {
+    }
+
+    /**
+     * Support iterator object.
+     */
+    RowList* RowList::__iter__() {
+        return this;
+    }
+
+    /**
+     * Support iterator object: get next row
+     */
+    void RowList::__next__(bool* hasRow) {
+        *hasRow = gsHasNextRow(mRowSet);
+        if (*hasRow) {
+            gsGetNextRow(mRowSet, mRow);
+        }
+    }
+
+    /**
+     * Refer GSRow pointer from RowSet
+     */
+    GSRow* RowList::get_gsrow_ptr() {
+        return this->mRow;
+    }
+
+    /**
+     * Refer GSType pointer from RowSet
+     */
+    GSType* RowList::get_gstype_list() {
+        return mTypelist;
+    }
+
+    /**
+     * Refer number column from RowSet
+     */
+    int RowList::get_column_count() {
+        return mColumnCount;
+    }
+
+    /**
+     * Refer number column from RowSet
+     */
+    bool RowList::get_timestamp_to_float() {
+        return mTimetampFloat;
+    }
+
+}  // namespace griddb

src/Query.h

@@ -0,0 +1,39 @@
+/*
+    Copyright (c) 2017 TOSHIBA Digital Solutions Corporation.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+#ifndef _ROWLIST_H_
+#define _ROWLIST_H_
+
+#include "gridstore.h"
+
+namespace griddb {
+class RowList {
+ private:
+    GSRowSet *mRowSet;
+    GSRow *mRow;
+    GSType* mTypelist;
+    int mColumnCount;
+    bool mTimetampFloat;
+ public:
+    RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType* typelist, int columnCount,
+            bool timetampFloat);
+    void __next__(bool* hasRow);
+    RowList* __iter__();
+    GSRow* get_gsrow_ptr();
+    GSType* get_gstype_list();
+    int get_column_count();
+    bool get_timestamp_to_float();
+};
+}  // namespace griddb
+
+#endif  // _ROWLIST_H_

src/RowList.cpp

@@ -239,7 +239,7 @@ namespace griddb {
             } catch (bad_alloc& ba) {
                 throw GSException(mRowSet, "Memory allocation error");
             }
-            
+
             for (int i = 0; i < mContainerInfo->columnCount; i++){
                 typeList[i] = mContainerInfo->columnInfoList[i].type;
             }
@@ -263,4 +263,19 @@ namespace griddb {
         return mRow;
     }
 
+    /**
+     * @brief Get row list data
+     * @param **row List row data
+     * @param **rowSet Rowset data
+     * @return A pointer store row list data in RowList object
+     */
+    griddb::RowList* RowSet::fetch_rows(GSRow **row, GSRowSet **rowSet) {
+        try {
+            return new RowList(this->getGSRowPtr(), this->mRowSet,
+                    this->getGSTypeList(), this->getColumnCount(),
+                    this->timestamp_output_with_float);
+        } catch (bad_alloc &ba) {
+            throw GSException(mRowSet, "Memory allocation error");
+        }
+    }
 }

src/RowList.h

@@ -29,6 +29,7 @@
 #include "QueryAnalysisEntry.h"
 #include "GSException.h"
 #include "Util.h"
+#include "RowList.h"
 
 using namespace std;
 
@@ -67,6 +68,7 @@ class RowSet {
         int getColumnCount();
 
         GSRow* getGSRowPtr();
+        griddb::RowList* fetch_rows(GSRow** row, GSRowSet** rowSet);
 
     private:
         RowSet(GSRowSet *rowSet, GSContainerInfo *containerInfo, GSRow *mRow);

src/RowSet.cpp

@@ -42,6 +42,7 @@
 %feature("new") griddb::Query::get_row_set;
 %feature("new") griddb::RowSet::get_next_query_analysis;
 %feature("new") griddb::RowSet::get_next_aggregation;
+%feature("new") griddb::RowSet::fetch_rows;
 %feature("new") griddb::Store::put_container;
 %feature("new") griddb::Store::get_container;
 %feature("new") griddb::Store::get_container_info;
@@ -83,6 +84,7 @@
 #include "RowKeyPredicate.h"
 #include "Store.h"
 #include "StoreFactory.h"
+#include "RowList.h"
 %}
 #if !defined(SWIGJAVASCRIPT)
 %{
@@ -109,6 +111,7 @@
 %shared_ptr(griddb::RowKeyPredicate)
 %shared_ptr(griddb::Store)
 %shared_ptr(griddb::PartitionController)
+%shared_ptr(griddb::RowList)
 #endif
 
 %include "GSException.h"
@@ -125,6 +128,7 @@
 %include "RowKeyPredicate.h"
 %include "Store.h"
 %include "StoreFactory.h"
+%include "RowList.h"
 #if !defined(SWIGJAVASCRIPT)
 %include "TimestampUtils.h"
 #endif