Merge pull request #662 from PecanProject/release/5.2.1

Release/5.2.1

Author: robkooper

.ruby-version

@@ -1 +1 @@
-ruby-2.3.0
+ruby-2.3.8

.travis.yml

@@ -7,8 +7,19 @@ sudo: false
 # versions of ruby to use
 # see the list at http://rubies.travis-ci.org/
 rvm:
-  - 2.2.4
-  - 2.3.0
+  - 2.2.9
+  - 2.3.8
+  - 2.4.6
+  - 2.5.5
+  - 2.6.3
+
+matrix:
+  fast_finish: true
+  allow_failures:
+    - rvm: 2.2.9
+    - rvm: 2.4.6
+    - rvm: 2.5.5
+    - rvm: 2.6.3
 
 # used for all ruby tasks to specify the database
 env:
@@ -25,7 +36,7 @@ addons:
   postgresql: 9.5
   apt:
     packages:
-      - postgresql-9.5-postgis-2.3
+      - postgresql-9.5-postgis-2.4
 
 # notifications should go to gitter
 notifications:
@@ -42,6 +53,9 @@ notifications:
 services:
   - postgresql
 
+install:
+  - bundle install --jobs=3 --retry=3 --with docker --without "javascript_testing"
+
 before_script:
   - psql -U postgres -c "CREATE EXTENSION postgis"
   - psql -U postgres -c "CREATE USER bety WITH SUPERUSER CREATEDB UNENCRYPTED PASSWORD 'bety'";

.zenodo.json

@@ -49,6 +49,10 @@
 		},
 		{
 			"name": "Chris Black"
+		},
+		{
+			"affiliation": "University of Arizona",
+			"name": "Chris Schnaufer"
 		}
 	],
 	"language": "eng",

CHANGELOG.md

@@ -5,6 +5,28 @@ section for the next release.
 
 For more information about this file see also [Keep a Changelog](http://keepachangelog.com/) .
 
+## [5.2.1] - 2019-08-07
+
+Currently the tested and recommended version of Ruby to be used with BETY is 2.3. BETY compiles and passes tests with 2.4, 2.5 and 2.6 but have not been tested thoroughly with these versions.
+
+### Added
+
+- #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. 
+
+### Changed
+
+- Documentation updates, now point to new [documentation location](https://pecanproject.github.io/bety-documentation/index.html)
+- #652 : Better list of soil choices
+- Ruby 2.3.8 is now default ruby when using RVM.
+
+### Fixed
+
+- Updated GEM list to fix security fixes as recommended by dependbot on GitHub.
+- Restores method assignment in Bulk Uploads
+- Fixed travis build, testing with 2.2, 2.3, 2.4, 2.5 and 2.6 of Ruby.
+- Fixed errors when running BETY with 2.4, 2.5 and 2.6
+- #668 : show "not found" page for invalid citations.
+
 ## [5.2.0] - 2019-03-11
 
 ### Added
@@ -15,7 +37,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha
 
 ## [5.1.0] - 2019-01-14
 
-### Fixes
+### Fixed
 
 - #611 : could not close alerts
 - #585 : Pressing "Show" on the Covariates Edit page attempts an update
@@ -26,34 +48,34 @@ For more information about this file see also [Keep a Changelog](http://keepacha
 
 ## [5.0.5] - 2018-10-23
 
-### Fixes
+### Fixed
 
 - #603 : Restores Rails route to make method completion work in Bulk Upload Wizard
 
 ## [5.0.4] - 2018-10-11
 
-### Fixes
+### Fixed
 
 - #600 : Error when starting BETY as docker container due to frozen variable.
 
 ## [5.0.3] - 2018-09-28
 
-### Fixes
+### Fixed
 
 - #593 : Replaces references to gitter with references to slack.
-- #598 : Fixes issue in docker where the entrypoint referenced a wrong URL to download the initial schema from.
+- #598 : Fixed issue in docker where the entrypoint referenced a wrong URL to download the initial schema from.
 
 ## [5.0.2] - 2018-08-09
 
-### Fixes
+### Fixed
 - Update to sprockets gem
 
 ## [5.0.1] - 2018-08-08
 
-### Fixes
-- Fixes fuzzy matching used in search queries
+### Fixed
+- Fixed fuzzy matching used in search queries
 
 ## [5.0.0] - 2018-08-07
 
-### Fixes
+### Fixed
 - Major upgrade to rails 5, make sure to read documentation on how to upgrade.

Gemfile

@@ -22,7 +22,7 @@ gem 'rails3-restful-authentication', '~> 3.0.1'
 gem 'dynamic_form'
 gem 'rabl'
 gem 'yajl-ruby', '~> 1.3.1'
-gem 'rubyzip', '~> 1.2.1'
+gem 'rubyzip', '~> 1.2.2'
 gem 'activerecord-session_store' # no longer part of Rails proper
 gem 'protected_attributes_continued' # Use this until and unless we start using Strong Parameters.
 gem 'sass-rails'

Gemfile.lock

@@ -63,7 +63,7 @@ GEM
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (>= 2.0, < 4.0)
-    capybara-webkit (1.15.0)
+    capybara-webkit (1.15.1)
       capybara (>= 2.3, < 4.0)
       json
     childprocess (0.9.0)
@@ -85,7 +85,7 @@ GEM
     dynamic_form (1.1.4)
     erubi (1.7.1)
     execjs (2.7.0)
-    ffi (1.9.23)
+    ffi (1.11.1)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
     i18n (0.9.5)
@@ -100,7 +100,7 @@ GEM
     json-schema (2.8.0)
       addressable (>= 2.4)
     kgio (2.11.2)
-    loofah (2.2.2)
+    loofah (2.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
@@ -114,7 +114,7 @@ GEM
     mustache (1.0.5)
     narray (0.6.0.4)
     nio4r (2.3.0)
-    nokogiri (1.8.2)
+    nokogiri (1.8.5)
       mini_portile2 (~> 2.3.0)
     passenger (5.3.4)
       rack
@@ -133,7 +133,7 @@ GEM
     public_suffix (3.0.2)
     rabl (0.13.1)
       activesupport (>= 2.3.14)
-    rack (2.0.5)
+    rack (2.0.7)
     rack-test (1.0.0)
       rack (>= 1.0, < 3)
     railroad (0.5.0)
@@ -202,7 +202,7 @@ GEM
       mustache (~> 1.0, >= 0.99.4)
       rspec (~> 3.0)
     ruby-graphviz (1.0.8)
-    rubyzip (1.2.1)
+    rubyzip (1.2.2)
     safe_attributes (1.0.10)
       activerecord (>= 3.0.0)
     sass (3.5.6)
@@ -245,7 +245,7 @@ GEM
     xpath (3.0.0)
       nokogiri (~> 1.8)
     yajl-ruby (1.3.1)
-    yard (0.9.12)
+    yard (0.9.20)
 
 PLATFORMS
   ruby
@@ -286,7 +286,7 @@ DEPENDENCIES
   rspec-rails (~> 3.0)
   rspec_api_documentation
   ruby-graphviz (= 1.0.8)
-  rubyzip (~> 1.2.1)
+  rubyzip (~> 1.2.2)
   safe_attributes
   sass-rails
   seer (= 0.10.0)
@@ -298,3 +298,6 @@ DEPENDENCIES
   will_paginate
   yajl-ruby (~> 1.3.1)
   yard
+
+BUNDLED WITH
+   1.17.3

README.md

@@ -35,7 +35,7 @@ docker run -e RAILS_RELATIVE_URL_ROOT="/bety" pecan/bety
 
 ## Documentation.
 
-* Technical Documentation: https://pecan.gitbooks.io/betydb-documentation/content/
-* Data Entry: https://pecan.gitbooks.io/betydbdoc-dataentry/content/
-* Data Access: https://pecan.gitbooks.io/betydb-data-access/content/
+* Technical Documentation: https://pecanproject.github.io/bety-documentation/technical/
+* Data Entry: https://pecanproject.github.io/bety-documentation/dataentry/
+* Data Access: https://pecan.gitbook.io/betydb-data-access/
 

app/controllers/citations_controller.rb

@@ -123,7 +123,10 @@ def index
   # GET /citations/1
   # GET /citations/1.xml
   def show
-    @citation = Citation.where(:id => params[:id]).includes(params[:include]).first
+    # find_by! throws an ActiveRecord::RecordNotFound exception if no citation
+    # with the given id exists so that we don't attempt to display a nil
+    # citation.
+    @citation = Citation.includes(params[:include]).find_by!(:id => params[:id])
 
     respond_to do |format|
       format.html # show.html.erb

app/models/bulk_upload_data_set.rb

@@ -1900,7 +1900,9 @@ def get_insertion_data
     # Get interactively-specified values, or set to empty hash if nil; since we
     # are going to alter interactively_specified_values, we use clone to make a
     # copy so that the session value remains as is.
-    interactively_specified_values = @session["global_values"].clone rescue ActionController::Parameters.new
+    interactively_specified_values = @session.has_key?(:global_values) \
+                                     ? @session[:global_values].clone \
+                                     : ActionController::Parameters.new
 
     # TO DO: decide if this code serves any useful purpose:
     # Double-check that all form fields are were non-empty:

app/models/site.rb

@@ -231,7 +231,7 @@ def city_state
 
   def sitename_state_country
     output = ""
-    
+
     #city = city.chomp if !city.nil?
     if !sitename.blank?
       output += "#{sitename}"
@@ -262,6 +262,17 @@ def autocomplete_label
     "#{sitename.squish} (#{city.squish}, #{!(state.nil? || state.empty?) ? " #{state.squish}," : ""} #{country.squish})"
   end
 
+  ### Class methods
+
+  def self.soil_classes
+
+    @soil_classes ||= ['clay', 'clay loam', 'loam', 'loamy sand', 'sand',
+                       'sandy clay', 'sandy clay loam', 'sandy loam',
+                       'silt', 'silt loam', 'silty clay', 'silty clay loam',
+                       'peat', 'bedrock', 'other'].freeze
+  end
+
+
   private
 
   ## Validation methods

app/views/layouts/application.html.erb

@@ -200,9 +200,9 @@
 
             <li class="menu-docs"> <a>Docs</a>
               <ul class="sub-menu">
-                <li><%= link_to "Data Entry", "https://pecan.gitbooks.io/betydbdoc-dataentry/content/", :target =>"_blank" %></li>
-                <li><%= link_to "Data Access", "https://pecan.gitbooks.io/betydb-data-access/content/", :target =>"_blank" %>></li>
-                <li><%= link_to "Implementation", "https://pecan.gitbooks.io/betydb-documentation/content/", :target =>"_blank" %></li>
+                <li><%= link_to "Data Entry", "https://pecanproject.github.io/bety-documentation/dataentry", :target =>"_blank" %></li>
+                <li><%= link_to "Data Access", "https://pecan.gitbook.io/betydb-data-access/", :target =>"_blank" %>></li>
+                <li><%= link_to "Implementation", "https://pecanproject.github.io/bety-documentation/technical/", :target =>"_blank" %></li>
                 <li><%= link_to "Schema", schemas_path %>></li>
               </ul>
             </li>
@@ -292,7 +292,7 @@
           <h3><strong>BETY</strong>db</h3>
           <ul class="unstyled">
                       <li><%= link_to "Homepage", root_path %></li>
-                      <li><%= link_to "Documentation", "https://pecan.gitbooks.io/betydb-documentation/content/", :target =>"_blank" %></li>
+                      <li><%= link_to "Documentation", "https://pecanproject.github.io/bety-documentation/", :target =>"_blank" %></li>
                       <% if CONFIG[:show_crop_map_links] %>
                       <li><%= link_to "Maps & Data", maps_path %></li>
                       <% end %>

app/views/sites/edit.html.erb

@@ -311,7 +311,7 @@
             <%= f.label :lon %>
             <%= geometry_aware_text_field(f, :lon, { class: "input-full" }) %>
             <%= f.label :soil %>
-            <%= f.select :soil, %w(sandloamy sandsandy loamsilt loamloamsandy clay loamsilty clay loamclay loamsandy claysilty clayclaypeat), { :include_blank => true }, :class => "input-full" %>
+            <%= f.select :soil, Site.soil_classes, { :include_blank => true }, :class => "input-full" %>
 
             <div class="two columns alpha">
               <%= f.label :clay_pct, "% Clay" %>

app/views/sites/new.html.erb

@@ -309,7 +309,7 @@
             <%= f.text_field :lon, :class => "input-full" %>
             <br /><br />
             <%= f.label :soil %>
-            <%= f.select :soil, %w(sandloamy sandsandy loamsilt loamloamsandy clay loamsilty clay loamclay loamsandy claysilty clayclaypeat), { :include_blank => true }, :class => "input-full" %>
+            <%= f.select :soil, Site.soil_classes, { :include_blank => true }, :class => "input-full" %>
 
             <div class="two columns alpha">
               <%= f.label :clay_pct, "% Clay" %>

docker/entrypoint.sh

@@ -23,6 +23,14 @@ case $1 in
         echo "Migrate database."
         rake db:migrate SKIP_SCHEMASPY=YES
         ;;
+    "reindex" )
+        echo "Reindexing database tables"
+        ./script/reindex.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" 
+        ;;
+    "reindex-all" )
+        echo "Reindexing entire database"
+        ./script/reindex.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -s
+        ;;
     "server" )
         echo "Start running BETY (rails server)"
         exec rails s
@@ -35,6 +43,18 @@ case $1 in
         echo "Start running BETY (unicorn)"
         exec bundle exec unicorn -c config/unicorn.rb
         ;;
+    "vacuum" )
+        echo "Vacuuming database tables"
+        ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -s
+        ;;
+    "vacuum-all" )
+        echo "Vacuuming entire database (not VACUUM FULL)"
+        ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres"
+        ;;
+    "vacuum-full" )
+        echo "Full vacuuming of entire database: VACUUM FULL"
+        ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -f
+        ;;
     "autoserver" )
         echo "Migrate database."
         rake db:migrate SKIP_SCHEMASPY=YES
@@ -50,8 +70,13 @@ case $1 in
         echo "sync       : synchronize database with remote servers ${REMOTE_SERVERS}"
         echo "dump       : dumps local database"
         echo "migrate    : migrates the database to a new version of bety"
+        echo "reindex    : maintentance: reindex the tables in the database"
+        echo "reindex-all: maintentance: reindex all of the database, do this sparingly"
         echo "server     : runs the server (using rails server)"
         echo "unicorn    : runs the server (using unicorn)"
+        echo "vacuum     : maintenance: vaccum the tables of the database"
+        echo "vacuum-all : maintenance: vaccum the entire database (not VACUUM FULL)"
+        echo "vacuum-full: maintenance: full vaccum of the database. Specify rarely, if ever"
         echo "autoserver : runs the server (using unicorn) after running a migrate"
         echo "help       : this text"
         echo ""

script/db_maintenance/README

@@ -15,7 +15,7 @@ This script was motivated by the need to repair a database where the id numbers
 for new rows were allocated in the wrong range—to be specific, they were
 allocated starting at 99000000001 rather than starting at the number
 corresponding to the id number of the machine where the database resides.  (See
-https://pecan.gitbook.io/betydb-documentation/betydb-system-administration/distributed-betydb#primary-key-allocations
+https://pecanproject.github.io/bety-documentation/technical/distributed-instances-of-betydb.html
 for a list of BETYdb instances and their corresponding primary key allocations.)
 
 When id numbers have been mis-allocated, three things must be done to repair the

script/load.bety.sh

@@ -250,7 +250,7 @@ ID_RANGE=1000000000
 # before anything is done, check to make sure database exists
 if ! psql ${PG_OPT} ${PG_USER} -lqt | cut -d \| -f 1 | grep -w "^ *${DATABASE} *$" > /dev/null ; then
   echo "Database ${DATABASE} does not exist, please create it:"
-  echo "(see https://pecan.gitbooks.io/betydb-documentation/content/installing_betydb.html)"
+  echo "(see https://pecanproject.github.io/bety-documentation/technical/deploying-a-production-copy-of-the-betydb-web-application.html)"
   echo "  psql ${PG_OPT} ${PG_USER} -c \"CREATE ROLE ${OWNER} WITH LOGIN CREATEDB NOSUPERUSER NOCREATEROLE PASSWORD 'password'\""
   echo "  psql ${PG_OPT} ${PG_USER} -c \"CREATE DATABASE ${DATABASE} WITH OWNER ${OWNER}\""
   exit 1