Merge branch 'pgspider:master' into draft_rpm_spec

Author: mkgrgis

.github/workflows/CI.yml

@@ -24,11 +24,12 @@ jobs:
   test:
     needs: detect-pgversion
     env:
-      SQLITE_VERSION : "3460000"
-      SQLITE_YEAR: "2024"
+      SQLITE_VERSION : "3490000"
+      SQLITE_YEAR: "2025"
       POSTGIS_VERSION : "3.4.2"
       HTTP_PROXY: ""
       HTTPS_PROXY: ""
+      SQLITE_FOR_TESTING_DIR: "/opt/CI_sqlite"
     strategy:
       fail-fast: false
       matrix:
@@ -46,32 +47,38 @@ jobs:
       - name: set_proxy
         run: bash GitHubActions/env.sh
 
+      - name: download PostGIS, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/download_postgis.sh ${{ env.POSTGIS_VERSION }}
+          fi
+
       - name: install locales
         run: bash GitHubActions/install_locales.sh
 
-      - name: build PostgreSQL ${{ matrix.pg }} with ${{ matrix.config }}
+      - name: build PostgreSQL ${{ matrix.pg }}
         run: bash GitHubActions/build_postgres.sh ${{ matrix.pg }}
 
-      - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
-        run: |
-          if [[ "${{ matrix.config }}" == "postgis" ]]; then
-            bash GitHubActions/build_postgis.sh ${{ matrix.pg }} ${{ env.POSTGIS_VERSION }}
-          fi
-
       - name: install SQLite, ${{ matrix.config }} mode
         run: |
           if [[ "${{ matrix.config }}" == "default" ]]; then
-            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} "${{ matrix.config }}"
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
           elif [[ "${{ matrix.config }}" == "postgis" ]]; then
-            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} "${{ matrix.config }}" --enable-rtree
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }} --enable-rtree
+          fi
+
+      - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/build_postgis.sh ${{ matrix.pg }} ${{ env.POSTGIS_VERSION }}
           fi
 
       - name: build sqlite_fdw, ${{ matrix.config }} mode
         run: |
-          bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }} ${{ matrix.config }}
+          bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
 
       - name: execute sqlite_fdw test
-        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }} ${{ matrix.config }}
+        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }} ${{ matrix.config }} ${{ env.SQLITE_FOR_TESTING_DIR }}
 
       - name: download output files (regression.diffs)
         if: failure()
@@ -83,3 +90,4 @@ jobs:
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/regression.out
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/results
           retention-days: 7
+

GitHubActions/build_postgis.sh

@@ -2,7 +2,7 @@
 
 ################################################################################
 #
-# This script downloads PostgreSQL from the official web site into ./workdir
+# This script downloads PostGIS from the official web site into ./workdir
 # then builds it.
 #
 # Usage: ./build_postgis.sh pg_version postgis_version
@@ -17,23 +17,18 @@
 POSTGRESQL_VERSION=$1
 POSTGIS_VERSION=$2
 
-cd ./workdir
-cd postgresql-${POSTGRESQL_VERSION}
-
 # Install necessary dependencies
 sudo apt update
 sudo apt install -y build-essential libxml2-dev libgeos-dev libproj-dev libgdal-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler
 
-GEOS_CONFIG_PATH=$(which geos-config)
-
+cd ./workdir
 # Download and compile PostGIS
-cd contrib
-wget http://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz
-tar -xzf postgis-${POSTGIS_VERSION}.tar.gz
-mv postgis-${POSTGIS_VERSION} postgis -v
-cd postgis
+cp -vr postgis postgresql-${POSTGRESQL_VERSION}/contrib
+cd postgresql-${POSTGRESQL_VERSION}/contrib/postgis
 echo " - PostGIS directory"
+GEOS_CONFIG_PATH=$(which geos-config)
 export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
 ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-geosconfig=$GEOS_CONFIG_PATH
 make
 sudo make install
+

GitHubActions/build_sqlite_fdw.sh

@@ -4,9 +4,10 @@
 #
 # This script builds sqlite_fdw in PostgreSQL source tree.
 #
-# Usage: ./build_sqlite_fdw.sh pg_version mode
+# Usage: ./build_sqlite_fdw.sh pg_version mode sqlite_for_testing_dir
 #     pg_version is a PostgreSQL version like 17.0 to be built in.
 #     mode is flag for sqlite_fdw compiler.
+#     sqlite_for_testing_dir: path to install directory of SQLite version for testing
 #
 # Requirements
 # - the source code of sqlite_fdw is available by git clone.
@@ -18,14 +19,16 @@ VERSION="$1"
 MODE="$2"
 
 mkdir -p ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-tar zxvf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
+tar zxf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+
+# show locally compiled sqlite library
+ls -la /usr/local/lib
 
 if [ "$MODE" == "postgis" ]; then
-  make ENABLE_GIS=1
+  make ENABLE_GIS=1 SQLITE_FOR_TESTING_DIR="$3"
 else
-  make
+  make SQLITE_FOR_TESTING_DIR="$3"
 fi
 
 sudo make install

GitHubActions/download_postgis.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+################################################################################
+#
+# This script downloads PostGIS from the official web site into ./workdir
+# then builds it.
+#
+# Usage: ./download_postgis.sh postgis_version
+#     postgis_version is a PostGIS version to be installed.
+#
+# Requirements
+# - be able to connect to the PostGIS official web site by wget.
+#
+################################################################################
+
+POSTGIS_VERSION=$1
+
+mkdir -p ./workdir
+cd ./workdir
+pgisfile="postgis-${POSTGIS_VERSION}.tar.gz"
+if [ ! -f "$pgisfile" ]; then
+  wget -nv "http://download.osgeo.org/postgis/source/$pgisfile"
+  tar -xzf "$pgisfile"
+  mv postgis-${POSTGIS_VERSION} postgis -v
+  echo "PostGIS source code directory " $(dirname $(readlink -f postgis))
+else
+  echo "PostGIS downloaded"
+fi

GitHubActions/execute_test.sh

@@ -6,9 +6,10 @@
 # sqlite_fdw. If all tests are passed, this script will exit successfully.
 # Otherwise, it will exit with failure.
 
-# Usage: ./execute_test.sh pg_version mode
+# Usage: ./execute_test.sh pg_version mode sqlite_for_testing_dir
 #     pg_version is a PostgreSQL version to be tested like 17.0.
 #	  mode is flag for sqlite_fdw compiler.
+#     sqlite_for_testing_dir: path to install directory of SQLite version for testing
 #
 # Requiremets
 # - the source code of PostgreSQL is located in ./workdir/postgresql-{pg_version}.
@@ -22,6 +23,7 @@
 
 VERSION=$1
 MODE="$2"
+SQLITE_FOR_TESTING_DIR="$3"
 
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
 

GitHubActions/install_locales.sh

@@ -17,5 +17,6 @@ sudo apt-get install locales language-pack-ja
 sudo locale-gen ja_JP.EUC-JP
 sudo apt-get install language-pack-ko-base language-pack-ko
 sudo locale-gen ko_KR.EUC-KR
-sudo apt -get install language-pack-bg-base language-pack-bg
+sudo apt-get install language-pack-bg-base language-pack-bg
 sudo locale-gen bg_BG
+sudo apt-get install libreadline8 libreadline-dev

GitHubActions/install_sqlite.sh

@@ -5,13 +5,14 @@
 # This sript downloads SQLite source code from the official web site into
 # ./workdir then builds and installs it.
 #
-# Usage: ./install_sqlite.sh version year testing_mode [configure_options]
+# Usage: ./install_sqlite.sh version year testing_mode sqlite_for_testing_dir [configure_options]
 #     version: SQLite version to be installed.
 #     year: A year of SQLite released. It is used for determining a download URL.
 #     testing_mode:	'default' or 'postgis' value.
+#     sqlite_for_testing_dir: path to install directory of the specified SQLite version
 #     configure_options: are a list of option for sqlite server.
 #
-#     Ex) ./install_sqlite.sh 3420000 2023 postgis --enable-rtree
+#     Ex) ./install_sqlite.sh 3420000 2023 postgis /opt/sqlite_for_testing --enable-rtree
 #
 # Requirements
 # - be able to connect to the SQLite official web site by curl.
@@ -22,27 +23,36 @@
 VERSION="$1"
 YEAR="$2"
 TESTING_MODE="$3"
+SQLITE_FOR_TESTING_DIR="$4"
 
 CONFIGURE_OPTIONS=""
 
 while (( "$#" )); do
-  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $4"
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $5"
   shift
 done
 
+echo "SQLite ver $VERSION ($YEAR), mode $TESTING_MODE, install to $SQLITE_FOR_TESTING_DIR with options $CONFIGURE_OPTIONS";
+
 mkdir -p ./workdir
 cd ./workdir
-curl -O https://www.sqlite.org/${YEAR}/sqlite-src-${VERSION}.zip
-unzip sqlite-src-${VERSION}.zip > /dev/null
-cd sqlite-src-${VERSION}
-
-if [ -z "$CONFIGURE_OPTIONS" ]; then
-  ./configure --enable-fts5
-else
-  ./configure --enable-fts5 $CONFIGURE_OPTIONS
+vsrc="sqlite-src-${VERSION}"
+adr="https://www.sqlite.org/${YEAR}/$vsrc.zip"
+echo "SQLite source code archive: $adr"
+wget "$adr" -O "$vsrc.zip"
+unzip "$vsrc.zip" > /dev/null
+cd "$vsrc"
+
+export CFLAGS=-DSQLITE_ENABLE_COLUMN_METADATA
+confcom="./configure --enable-fts5 --prefix=$SQLITE_FOR_TESTING_DIR"
+if [ ! -z "$CONFIGURE_OPTIONS" ]; then
+  confcom+="$CONFIGURE_OPTIONS"  
 fi
+echo "SQLite configure call: $confcom"
+$confcom
 
 make
+echo "----- SQLITE INSTALL directory $SQLITE_FOR_TESTING_DIR -----"
 sudo make install
 
 if [ "$TESTING_MODE" == "postgis" ]; then

Makefile

@@ -16,20 +16,29 @@ EXTENSION = sqlite_fdw
 DATA = sqlite_fdw--1.0.sql sqlite_fdw--1.0--1.1.sql
 
 ifdef ENABLE_GIS
-override PG_CFLAGS += -DSQLITE_FDW_GIS_ENABLE
-GISTEST=postgis
+PG_CFLAGS += -DSQLITE_FDW_GIS_ENABLE
+GIS_DEP_TESTS_DIR = with_gis_support
+GIS_TEST = postgis
+$(info  There is PostGIS support for SQLite FDW)
 else
-GISTEST=nogis
+GIS_DEP_TESTS_DIR = without_gis_support
+GIS_TEST = nogis
+$(info  There is NO PostGIS support for SQLite FDW)
 endif
 
+# Tests for PostgreSQL data types support
+DATA_TYPE_TESTS = types/bitstring types/bool types/float4 types/float8 types/int4 types/int8 types/json types/numeric  types/macaddr types/macaddr8 types/out_of_range types/timestamp types/uuid
+# Tests with different versions with GIS support and without GIS support
+GIS_DEP_TESTS = $(GIS_DEP_TESTS_DIR)/type $(GIS_DEP_TESTS_DIR)/auto_import $(GIS_DEP_TESTS_DIR)/$(GIS_TEST)
+
 ifndef REGRESS
-REGRESS = extra/sqlite_fdw_post types/bitstring types/bool types/float4 types/float8 types/int4 types/int8 types/numeric types/$(GISTEST) types/macaddr types/macaddr8 types/out_of_range types/timestamp types/uuid extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw type_$(GISTEST) aggregate selectfunc 
+# System tests, full default sequence
+REGRESS = libsqlite extra/sqlite_fdw_post $(DATA_TYPE_TESTS) extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/encodings sqlite_fdw aggregate selectfunc $(GIS_DEP_TESTS)
 endif
 
+# Other encodings also are tested. Client encoding should be UTF-8.
 REGRESS_OPTS = --encoding=utf8
 
-SQLITE_LIB = sqlite3
-
 UNAME = uname
 OS := $(shell $(UNAME))
 ifeq ($(OS), Darwin)
@@ -38,7 +47,12 @@ else
 DLSUFFIX = .so
 endif
 
+ifdef SQLITE_FOR_TESTING_DIR
+SHLIB_LINK := -L$(SQLITE_FOR_TESTING_DIR)/lib -lsqlite3
+PG_CFLAGS += -I$(SQLITE_FOR_TESTING_DIR)/include -Wl,-rpath,$(SQLITE_FOR_TESTING_DIR)/lib
+else
 SHLIB_LINK := -lsqlite3
+endif
 
 ifdef ENABLE_GIS
 override SHLIB_LINK += -lspatialite
@@ -70,9 +84,19 @@ endif
 REGRESS := $(addprefix $(REGRESS_PREFIX_SUB)/,$(REGRESS))
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/extra)
 $(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/types)
+$(shell mkdir -p results/$(REGRESS_PREFIX_SUB)/$(GIS_DEP_TESTS_DIR))
+
+# $(info    ENABLE_GIS      is $(ENABLE_GIS))
+# $(info    SHLIB_LINK      is $(SHLIB_LINK))
+# $(info    LD_LIBRARY_PATH is $(LD_LIBRARY_PATH))
+# $(info    PG_CFLAGS       is $(PG_CFLAGS))
+# $(info    PG_CPPFLAGS     is $(PG_CPPFLAGS))
+# $(info    REGRESS         is $(REGRESS))
+# $(info    DLSUFFIX        is $(DLSUFFIX))
 
 ifdef ENABLE_GIS
 check: temp-install
 temp-install: EXTRA_INSTALL+=contrib/postgis
 checkprep: EXTRA_INSTALL+=contrib/postgis
 endif
+