Merge pull request #1001 from dr-itz/50-dev

[perf] set a fetch size to reduce peak memory usage

Author: enebo

src/java/arjdbc/jdbc/RubyJdbcConnection.java

@@ -128,6 +128,7 @@ public class RubyJdbcConnection extends RubyObject {
     private boolean lazy = false; // final once set on initialize
     private boolean jndi; // final once set on initialize
     private boolean configureConnection = true; // final once initialized
+    private int fetchSize = 0; // 0 = JDBC default
 
     protected RubyJdbcConnection(Ruby runtime, RubyClass metaClass) {
         super(runtime, metaClass);
@@ -571,6 +572,11 @@ protected void doInitialize(final ThreadContext context, final IRubyObject confi
         else {
             this.configureConnection = value != context.runtime.getFalse();
         }
+
+        IRubyObject jdbcFetchSize = getConfigValue(context, "jdbc_fetch_size");
+        if (jdbcFetchSize != context.nil) {
+            this.fetchSize = RubyNumeric.fix2int(jdbcFetchSize);
+        }
     }
 
     @JRubyMethod(name = "adapter")
@@ -818,6 +824,7 @@ public IRubyObject call(final Connection connection) throws SQLException {
                             // is called, so we have to process the result sets as we get them
                             // this shouldn't be an issue in most cases since we're only getting 1 result set anyways
                             result = mapExecuteResult(context, connection, resultSet);
+                            resultSet.close();
                         } else {
                             result = context.runtime.newFixnum(updateCount);
                         }
@@ -851,6 +858,7 @@ protected Statement createStatement(final ThreadContext context, final Connectio
         else {
             statement.setEscapeProcessing(escapeProcessing.isTrue());
         }
+        if (fetchSize != 0) statement.setFetchSize(fetchSize);
         return statement;
     }
 
@@ -1045,6 +1053,7 @@ public IRubyObject call(final Connection connection) throws SQLException {
                     else {
                         final PreparedStatement prepStatement;
                         statement = prepStatement = connection.prepareStatement(query);
+                        if (fetchSize != 0) statement.setFetchSize(fetchSize);
                         statement.setMaxRows(maxRows); // zero means there is no limit
                         setStatementParameters(context, connection, prepStatement, binds);
                         hasResult = prepStatement.execute();
@@ -1123,7 +1132,9 @@ public IRubyObject prepare_statement(final ThreadContext context, final IRubyObj
         return withConnection(context, new Callable<IRubyObject>() {
             public IRubyObject call(Connection connection) throws SQLException {
                 final String query = sql.convertToString().getUnicodeValue();
-                return JavaUtil.convertJavaToRuby(context.runtime, connection.prepareStatement(query));
+                PreparedStatement statement = connection.prepareStatement(query);
+                if (fetchSize != 0) statement.setFetchSize(fetchSize);
+                return JavaUtil.convertJavaToRuby(context.runtime, statement);
             }
         });
     }
@@ -1158,19 +1169,15 @@ public IRubyObject call(final Connection connection) throws SQLException {
                         statement = (PreparedStatement) JavaEmbedUtils.rubyToJava(cachedStatement);
                     } else {
                         statement = connection.prepareStatement(query);
+                        if (fetchSize != 0) statement.setFetchSize(fetchSize);
                     }
 
                     setStatementParameters(context, connection, statement, (RubyArray) binds);
 
                     if (statement.execute()) {
                         ResultSet resultSet = statement.getResultSet();
                         IRubyObject results = mapQueryResult(context, connection, resultSet);
-
-                        if (cached) {
-                            // Make sure we free the result set if we are caching the statement
-                            // It gets closed automatically when the statement is closed if we aren't caching
-                            resultSet.close();
-                        }
+                        resultSet.close();
 
                         return results;
                     } else {
@@ -2501,6 +2508,8 @@ protected IRubyObject arrayToRuby(final ThreadContext context,
             while ( arrayResult.next() ) {
                 array.append( jdbcToRuby(context, runtime, 2, baseType, arrayResult) );
             }
+            arrayResult.close();
+
             return array;
         }
         finally { if ( value != null ) value.free(); }

test/rails/config.yml

@@ -62,6 +62,7 @@ connections:
       driver: <%= 'org.mariadb.jdbc.Driver' %>
       <% end %>
       <% if defined? JRUBY_VERSION %>
+      jdbc_fetch_size: 100
       properties:
         serverTimezone: <%= java.util.TimeZone.getDefault.getID %>
       <% end %>
@@ -80,6 +81,7 @@ connections:
       prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || ENV['PS'] %> # should leave it off to its default if not set
       <% end %>
       username: <%= ENV['PGUSER'] || ENV['user'] || 'rails' %>
+      jdbc_fetch_size: 100
       properties:
         # Postgres' JDBC driver does not prepare statements until executed 5 times by default, let's lower:
         prepareThreshold: <%= ENV['PREPARE_THRESHOLD'] || 1 %>
@@ -96,9 +98,11 @@ connections:
     arunit:
       database: <%= FIXTURES_ROOT %>/fixture_database.sqlite3
       timeout:  5000
+      jdbc_fetch_size: 100
     arunit2:
       database: <%= FIXTURES_ROOT %>/fixture_database_2.sqlite3
       timeout:  5000
+      jdbc_fetch_size: 100
 
   sqlite3_mem:
     arunit:

test/rails/excludes/mysql2/ActiveRecord/ConnectionAdapters/ReaperTest.rb

@@ -0,0 +1,2 @@
+exclude :test_some_time, 'intermittent failures, leaks thread, fires at high frequency'
+exclude :test_connection_pool_starts_reaper, 'intermittent failures, leaks thread, fires at high frequency'

test/rails/excludes/postgresql/ActiveRecord/ConnectionAdapters/ReaperTest.rb

@@ -0,0 +1,2 @@
+exclude :test_some_time, 'intermittent failures, leaks thread, fires at high frequency'
+exclude :test_connection_pool_starts_reaper, 'intermittent failures, leaks thread, fires at high frequency'

test/rails/excludes/sqlite3/ActiveRecord/ConnectionAdapters/ReaperTest.rb

@@ -0,0 +1,2 @@
+exclude :test_some_time, 'intermittent failures, leaks thread, fires at high frequency'
+exclude :test_connection_pool_starts_reaper, 'intermittent failures, leaks thread, fires at high frequency'