Movable DataBase Locales for Cloudberry

We inherited this issue from PostgreSQL.

PostgreSQL uses glibc to sort strings. In version glibc=2.28, collations
broke down badly (in general, there are no guarantees when updating glibc).
Changing collations breaks indexes. Similarly, a cluster with different
collations also behaves unpredictably.

What and when something has changed in glibc can be found
on https://github.com/ardentperf/glibc-unicode-sorting
Also there is special postgresql-wiki https://wiki.postgresql.org/wiki/Locale_data_changes
And you tube video https://www.youtube.com/watch?v=0E6O-V8Jato

In short, the issue can be seen through the use of bash:

( echo "1-1"; echo "11" ) | LC_COLLATE=en_US.UTF-8 sort

gives the different results in ubunru 18.04 and 22.04.

There is no way to solve the problem other than by not changing the symbol order.
We freeze symbol order and use it instead of glibc.

Here the solution https://github.com/postgredients/mdb-locales.

In this PR I have added PostgreSQL patch that replaces all glibc
locale-related calls with a calls to an external libary. It activates
using new configure parameter --with-mdblocales, which is off by
default.

Using custom locales needs libmdblocales1 package and mdb-locales
package with symbol table.

Build needs libmdblocales-dev package with headers.

Author: usernamedt

configure

@@ -698,6 +698,7 @@ BISON
 MKDIR_P
 LN_S
 TAR
+USE_MDBLOCALES
 install_bin
 INSTALL_DATA
 INSTALL_SCRIPT
@@ -943,6 +944,7 @@ with_rt
 with_libcurl
 with_apr_config
 with_gnu_ld
+with_mdblocales
 with_ssl
 with_openssl
 enable_openssl_redirect
@@ -1690,6 +1692,7 @@ Optional Packages:
   --without-libcurl       do not use libcurl
   --with-apr-config=PATH  path to apr-1-config utility
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
+  --without-mdblocales    build without MDB locales
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
 
@@ -2906,7 +2909,6 @@ PG_PACKAGE_VERSION=14.4
 
 
 
-
 ac_aux_dir=
 for ac_dir in config "$srcdir"/config; do
   if test -f "$ac_dir/install-sh"; then
@@ -12124,6 +12126,38 @@ case $INSTALL in
 esac
 
 
+#
+# MDB locales
+#
+
+
+
+
+# Check whether --with-mdblocales was given.
+if test "${with_mdblocales+set}" = set; then :
+  withval=$with_mdblocales;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_MDBLOCALES 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-mdblocales option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_mdblocales=no
+
+fi
+
+
+
+
 if test -z "$TAR"; then
   for ac_prog in tar
 do
@@ -12760,6 +12794,56 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; }
 
 
 
+fi
+
+if test "$with_mdblocales" = yes; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5
+$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; }
+if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmdblocales  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mdb_setlocale ();
+int
+main ()
+{
+return mdb_setlocale ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_mdblocales_mdb_setlocale=yes
+else
+  ac_cv_lib_mdblocales_mdb_setlocale=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5
+$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; }
+if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBMDBLOCALES 1
+_ACEOF
+
+  LIBS="-lmdblocales $LIBS"
+
+else
+  as_fn_error $? "mdblocales library not found" "$LINENO" 5
+fi
+
 fi
 
 if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then
@@ -16981,6 +17065,17 @@ fi
 
 done
 
+fi
+
+if test "$with_mdblocales" = yes; then
+  ac_fn_c_check_header_mongrel "$LINENO" "mdblocales.h" "ac_cv_header_mdblocales_h" "$ac_includes_default"
+if test "x$ac_cv_header_mdblocales_h" = xyes; then :
+
+else
+  as_fn_error $? "mdblocales header not found." "$LINENO" 5
+fi
+
+
 fi
 
 if test "$with_gssapi" = yes ; then

configure.ac

@@ -1448,6 +1448,14 @@ case $INSTALL in
 esac
 AC_SUBST(install_bin)
 
+#
+# MDB locales
+#
+
+PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales],
+              [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])])
+AC_SUBST(USE_MDBLOCALES)
+
 PGAC_PATH_PROGS(TAR, tar)
 AC_PROG_LN_S
 AC_PROG_MKDIR_P
@@ -1606,6 +1614,11 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_mdblocales" = yes; then
+  AC_CHECK_LIB(mdblocales, mdb_setlocale, [],
+               [AC_MSG_ERROR([mdblocales library not found])])
+fi
+
 if test "$enable_external_fts" = yes; then
 AC_CHECK_LIB(jansson, jansson_version_str, [],
                [AC_MSG_ERROR([jansson library not found or version is too old, version must >= 2.13])])
@@ -1985,6 +1998,10 @@ if test "$with_lz4" = yes; then
   AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])
 fi
 
+if test "$with_mdblocales" = yes; then
+  AC_CHECK_HEADER(mdblocales.h, [], [AC_MSG_ERROR([mdblocales header not found.])])
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])

contrib/pax_storage/src/cpp/storage/oper/pax_oper.cc

@@ -25,6 +25,7 @@
  *-------------------------------------------------------------------------
  */
 
+#include "common/mdb_locale.h"
 #include "storage/oper/pax_oper.h"
 
 #include "comm/cbdb_wrappers.h"
@@ -588,9 +589,9 @@ static inline bool LocaleIsC(Oid collation) {
       return (bool)result;
     }
 
-    localeptr = setlocale(LC_COLLATE, NULL);
+    localeptr = SETLOCALE(LC_COLLATE, NULL);
     CBDB_CHECK(localeptr, cbdb::CException::ExType::kExTypeCError,
-               fmt("Invalid locale, fail to `setlocale`, errno: %d", errno));
+               fmt("Invalid locale, fail to `SETLOCALE`, errno: %d", errno));
 
     if (strcmp(localeptr, "C") == 0 ||  // cut line
         strcmp(localeptr, "POSIX") == 0) {

devops/build/automation/cloudberry/scripts/configure-cloudberry.sh

@@ -61,6 +61,12 @@
 #                   --enable-cassert
 #                   --enable-debug-extensions
 #
+#   ENABLE_MDBLOCALES - Enable custom locales (true/false, defaults to
+#                       false)
+#
+#                 When true, add option:
+#                   --with-mdblocales
+#
 # Prerequisites:
 #   - System dependencies must be installed:
 #     * xerces-c development files
@@ -125,6 +131,11 @@ if [ "${ENABLE_DEBUG:-false}" = "true" ]; then
                           --enable-debug-extensions"
 fi
 
+CONFIGURE_MDBLOCALES_OPTS="--without-mdblocales"
+if [ "${ENABLE_MDBLOCALES:-false}" = "true" ]; then
+    CONFIGURE_MDBLOCALES_OPTS="--with-mdblocales"
+fi
+
 # Configure build
 log_section "Configure"
 execute_cmd ./configure --prefix=/usr/local/cloudberry-db \
@@ -151,6 +162,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \
             --with-ssl=openssl \
             --with-openssl \
             --with-uuid=e2fs \
+            ${CONFIGURE_MDBLOCALES_OPTS} \
             --with-includes=/usr/local/xerces-c/include \
             --with-libraries=/usr/local/cloudberry-db/lib || exit 4
 log_section_end "Configure"

gpcontrib/orafce/others.c

@@ -45,6 +45,7 @@
 #include "utils/uuid.h"
 #include "orafce.h"
 #include "builtins.h"
+#include "common/mdb_locale.h"
 
 /*
  * Source code for nlssort is taken from postgresql-nls-string
@@ -322,7 +323,7 @@ _nls_run_strxfrm(text *string, text *locale)
 	 */
 	if (!lc_collate_cache)
 	{
-		if ((lc_collate_cache = setlocale(LC_COLLATE, NULL)))
+		if ((lc_collate_cache = SETLOCALE(LC_COLLATE, NULL)))
 			/* Make a copy of the locale name string. */
 #ifdef _MSC_VER
 			lc_collate_cache = _strdup(lc_collate_cache);
@@ -364,7 +365,7 @@ _nls_run_strxfrm(text *string, text *locale)
 			 * If setlocale failed, we know the default stayed the same,
 			 * co we can safely elog.
 			 */
-			if (!setlocale(LC_COLLATE, locale_str))
+			if (!SETLOCALE(LC_COLLATE, locale_str))
 				elog(ERROR, "failed to set the requested LC_COLLATE value [%s]", locale_str);
 
 			changed_locale = true;
@@ -409,7 +410,7 @@ _nls_run_strxfrm(text *string, text *locale)
 			/*
 			 * Set original locale
 			 */
-			if (!setlocale(LC_COLLATE, lc_collate_cache))
+			if (!SETLOCALE(LC_COLLATE, lc_collate_cache))
 				elog(FATAL, "failed to set back the default LC_COLLATE value [%s]", lc_collate_cache);
 		}
 
@@ -422,7 +423,7 @@ _nls_run_strxfrm(text *string, text *locale)
 		/*
 		 * Set original locale
 		 */
-		if (!setlocale(LC_COLLATE, lc_collate_cache))
+		if (!SETLOCALE(LC_COLLATE, lc_collate_cache))
 			elog(FATAL, "failed to set back the default LC_COLLATE value [%s]", lc_collate_cache);
 		pfree(locale_str);
 	}

src/backend/gporca/libgpos/server/src/unittest/gpos/string/CWStringTest.cpp

@@ -12,6 +12,7 @@
 #include "unittest/gpos/string/CWStringTest.h"
 
 #include <locale.h>
+#include "common/mdb_locale.h"
 
 #include "gpos/base.h"
 #include "gpos/error/CAutoTrace.h"
@@ -177,18 +178,18 @@ CWStringTest::EresUnittest_AppendFormatInvalidLocale()
 	CWStringDynamic *expected =
 		GPOS_NEW(mp) CWStringDynamic(mp, GPOS_WSZ_LIT("UNKNOWN"));
 
-	CHAR *oldLocale = setlocale(LC_CTYPE, nullptr);
+	CHAR *oldLocale = SETLOCALE(LC_CTYPE, nullptr);
 	CWStringDynamic *pstr1 = GPOS_NEW(mp) CWStringDynamic(mp);
 
 	GPOS_RESULT eres = GPOS_OK;
 
-	setlocale(LC_CTYPE, "C");
+	SETLOCALE(LC_CTYPE, "C");
 	pstr1->AppendFormat(GPOS_WSZ_LIT("%s"), (CHAR *) "ÃË", 123);
 
 	pstr1->Equals(expected);
 
 	// cleanup
-	setlocale(LC_CTYPE, oldLocale);
+	SETLOCALE(LC_CTYPE, oldLocale);
 	GPOS_DELETE(pstr1);
 	GPOS_DELETE(expected);
 

src/backend/utils/adt/Makefile

@@ -117,7 +117,8 @@ OBJS = \
 	windowfuncs.o \
 	xid.o \
 	xid8funcs.o \
-	xml.o
+	xml.o \
+	mdb.o
 
 jsonpath_scan.c: FLEXFLAGS = -CF -p -p
 jsonpath_scan.c: FLEX_NO_BACKUP=yes

src/backend/utils/adt/mdb.c

@@ -0,0 +1,37 @@
+/*-------------------------------------------------------------------------
+ *
+ * mdb.c
+ *	  mdb routines
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/utils/adt/mdb.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+
+#include "postgres.h"
+#include "fmgr.h"
+#include "utils/fmgrprotos.h"
+
+/*
+ * mdb_admin_enabled
+ *		Check that mdb locale patch is enabled
+ */
+Datum
+mdb_locale_enabled(PG_FUNCTION_ARGS)
+{
+	bool res;
+
+#if USE_MDBLOCALES
+	res = true;
+#else
+	res = false;
+#endif
+
+	PG_RETURN_BOOL(res);
+}