Initial PostgreSQL support

.env.dist

@@ -24,7 +24,7 @@ DATABASE_URL=sqlite:///%kernel.project_dir%/var/data/bolt.sqlite
 #DATABASE_URL=mysql://db_user:"db_password"@localhost:3306/db_name
 
 # Postgres
-#DATABASE_URL=postgresql://db_user:"db_password"@localhost:5432/db_name?serverVersion=11"
+#DATABASE_URL=postgresql://db_user:"db_password"@localhost:5432/db_name?serverVersion=11&charset=utf8"
 
 # MYSQL / MariaDB (additional settings, needed for Docker)
 #DATABASE_USER=db_user

.gitignore

@@ -92,3 +92,7 @@ appveyor.yml
 ###> phpunit/phpunit ###
 /phpunit.xml
 ###< phpunit/phpunit ###
+
+###> .preload dev ###
+/src/.preload.php
+###< .preload dev ###

README.md

@@ -117,7 +117,19 @@ bin/console doctrine:fixtures:load -n
 
 Alternatively, run `make db-reset`, on a UNIX-like system.
 
-4 Run the prototype
+4 How to build assets
+-------------------
+
+To set up initially, run `npm install` to get the required dependencies /
+`node_modules`. Alternatively give the path to the python executable (`npm install --python="/usr/local/bin/python3.7"`) Then:
+  - Prepare directory structure `mkdir -p node_modules/node-sass/vendor`
+  - Rebuild npm environment for current OS `npm rebuild node-sass`
+  - Run `npm run start` (alternatively `npm run start --python="/usr/local/bin/python3.7"`)
+
+See the other options by running `npm run`.
+(Note: as I'm testing this as well remotely, I copied all assets from the released composer installation by `cp -r ../www_backup/public/assets/* public/assets/`)
+
+5 Run the prototype
 -------------------
 
   - Using the Symfony CLI tool, just run `symfony server:start`.
@@ -134,17 +146,6 @@ You can log on, using the default user & pass:
  - pass: `admin%1`
 
 
-How to build assets
--------------------
-
-To set up initially, run `npm install` to get the required dependencies /
-`node_modules`. Then:
-
-  - Run `npm run start`
-
-See the other options by running `npm run`.
-
-
 Code Style checking / Static Analysis
 ----------------------------
 

config/packages/doctrine.yaml

@@ -32,7 +32,8 @@ doctrine:
         dql:
             string_functions:
                 JSON_EXTRACT: Bolt\Doctrine\Functions\JsonExtract
-                CAST: DoctrineExtensions\Query\Mysql\Cast
+                JSON_GET_TEXT: Scienta\DoctrineJsonFunctions\Query\AST\Functions\Postgresql\JsonGetText
+                CAST: Bolt\Doctrine\Query\Cast
             numeric_functions:
                 RAND: Bolt\Doctrine\Functions\Rand
 

src/Doctrine/Functions/Rand.php

@@ -19,6 +19,10 @@ public function getSql(\Doctrine\ORM\Query\SqlWalker $sqlWalker)
         if (property_exists($this->expression, 'value') && $this->expression->value === '1') {
             return 'random()';
         }
+        // value is two if PostgreSQL. See Bolt\Storage\Directive\RandomDirectiveHandler
+        if (property_exists($this->expression, 'value') && $this->expression->value === '2') {
+            return 'RANDOM()';
+        }
 
         return 'RAND()';
     }

src/Doctrine/JsonHelper.php

@@ -25,7 +25,14 @@ public static function wrapJsonFunction(?string $where = null, ?string $slug = n
         $version = new Version($connection);
 
         if ($version->hasJson()) {
-            $resultWhere = 'JSON_EXTRACT(' . $where . ", '$[0]')";
+            //PostgreSQL handles JSON differently than MySQL
+            if ($version->getPlatform()['driver_name'] === 'pgsql') {
+                // PostgreSQL
+                $resultWhere = 'JSON_GET_TEXT(' . $where . ', 0)';
+            } else {
+                // MySQL and SQLite
+                $resultWhere = 'JSON_EXTRACT(' . $where . ", '$[0]')";
+            }
             $resultSlug = $slug;
         } else {
             $resultWhere = $where;

src/Doctrine/Query/Cast.php

@@ -0,0 +1,54 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Bolt\Doctrine\Query;
+
+use Doctrine\ORM\Query\AST\Functions\FunctionNode;
+use Doctrine\ORM\Query\Lexer;
+use Doctrine\ORM\Query\Parser;
+use Doctrine\ORM\Query\SqlWalker;
+
+class Cast extends FunctionNode
+{
+    /** @var \Doctrine\ORM\Query\AST\PathExpression */
+    protected $first;
+    /** @var string */
+    protected $second;
+    /** @var string */
+    protected $backend_driver;
+
+    public function getSql(SqlWalker $sqlWalker): string
+    {
+        $backend_driver = $sqlWalker->getConnection()->getDriver()->getName();
+
+        // test if we are using MySQL
+        if (mb_strpos($backend_driver, 'mysql') !== false) {
+            // YES we are using MySQL
+            // how do we know what type $this->first is? For now hardcoding
+            // type(t.value) = JSON for MySQL. JSONB for others.
+            // alternatively, test if true: $this->first->dispatch($sqlWalker)==='b2_.value',
+            // b4_.value for /bolt/new/showcases
+            if ($this->first->dispatch($sqlWalker) === 'b2_.value' ||
+                 $this->first->dispatch($sqlWalker) === 'b4_.value') {
+                return $this->first->dispatch($sqlWalker);
+            }
+        }
+
+        return sprintf('CAST(%s AS %s)',
+            $this->first->dispatch($sqlWalker),
+            $this->second
+            );
+    }
+
+    public function parse(Parser $parser): void
+    {
+        $parser->match(Lexer::T_IDENTIFIER);
+        $parser->match(Lexer::T_OPEN_PARENTHESIS);
+        $this->first = $parser->ArithmeticPrimary();
+        $parser->match(Lexer::T_AS);
+        $parser->match(Lexer::T_IDENTIFIER);
+        $this->second = $parser->getLexer()->token['value'];
+        $parser->match(Lexer::T_CLOSE_PARENTHESIS);
+    }
+}

src/Doctrine/Version.php

@@ -9,22 +9,27 @@
 use Doctrine\DBAL\Driver\PDOConnection;
 use Doctrine\DBAL\Platforms\MariaDb1027Platform;
 use Doctrine\DBAL\Platforms\MySQL57Platform;
+use Doctrine\DBAL\Platforms\PostgreSQL92Platform;
 use Doctrine\DBAL\Platforms\SqlitePlatform;
 
 class Version
 {
     /**
      * We're g̶u̶e̶s̶s̶i̶n̶g̶ doing empirical research on which versions of SQLite
      * support JSON. So far, tests indicate:
+     * https://www.sqlite.org/json1.html --> JSON since SQLite version 3.9.0 (2015-10-14)
+     * - 3.32.2 - OK (Wytse's FBSD 12.1 \w PHP 7.2)
      * - 3.20.1 - Not OK (Travis PHP 7.2)
      * - 3.27.2 - OK (Bob's Raspberry Pi, running PHP 7.3.11 on Raspbian)
      * - 3.28.0 - OK (Travis PHP 7.3)
      * - 3.28.0 - Not OK (Bob's PHP 7.2, installed with Brew)
      * - 3.29.0 - OK (MacOS Mojave)
      * - 3.30.1 - OK (MacOS Catalina)
      */
-    public const SQLITE_WITH_JSON = '3.27.0';
-    public const PHP_WITH_SQLITE = '7.3.0';
+    // JSON supported since SQLite version 3.9.0
+    public const SQLITE_WITH_JSON = '3.9.0';
+    // PHP supports SQLite since version 5.3.0
+    public const PHP_WITH_SQLITE = '5.3.0';
 
     /** @var Connection */
     private $connection;
@@ -87,6 +92,11 @@ public function hasJson(): bool
             return true;
         }
 
+        // PostgreSQL supports JSON from v9.2 and above, later versions are implicitly included
+        if ($platform instanceof PostgreSQL92Platform) {
+            return true;
+        }
+
         return false;
     }
 

src/Entity/FieldTranslation.php

@@ -21,7 +21,7 @@ class FieldTranslation implements TranslationInterface
      */
     private $id;
 
-    /** @ORM\Column(type="json") */
+    /** @ORM\Column(type="json", options={"jsonb": true}) */
     protected $value = [];
 
     public function getId(): ?int

src/Entity/Log.php

@@ -38,7 +38,7 @@ class Log
     /** @ORM\Column(name="extra", type="array", nullable=true) */
     private $extra;
 
-    /** @ORM\Column(name="user", type="array", nullable=true) */
+    /** @ORM\Column(name="`user`", type="array", nullable=true) */
     private $user;
 
     /** @ORM\Column(type="content", type="integer", nullable=true) */

src/Repository/ContentRepository.php

@@ -94,10 +94,14 @@ public function searchNaive(string $searchTerm, int $page, int $amountPerPage, C
         $qb = $this->getQueryBuilder()
             ->select('partial content.{id}');
 
+        // proper JSON wrapping solves a lot of problems (added PostgreSQL compatibility)
+        $connection = $qb->getEntityManager()->getConnection();
+        [$where, $searchTerm] = JsonHelper::wrapJsonFunction('t.value', $searchTerm, $connection);
+
         $qb->addSelect('f')
             ->innerJoin('content.fields', 'f')
             ->innerJoin('f.translations', 't')
-            ->andWhere($qb->expr()->like('t.value', ':search'))
+            ->andWhere($qb->expr()->like($where, ':search'))
             ->setParameter('search', '%' . $searchTerm . '%');
 
         // These are the ID's of content we need.

src/Storage/Directive/OrderDirective.php

@@ -115,9 +115,10 @@ private function setOrderBy(QueryInterface $query, string $order, string $direct
             } else {
                 // Note the `lower()` in the `addOrderBy()`. It is essential to sorting the
                 // results correctly. See also https://github.com/bolt/core/issues/1190
+                // again: lower breaks postgresql jsonb compatibility, first cast as txt
                 $query
                     ->getQueryBuilder()
-                    ->addOrderBy('lower(' . $translationsAlias . '.value)', $direction);
+                    ->addOrderBy('lower(CAST(' . $translationsAlias . '.value as TEXT))', $direction);
             }
             $query->incrementIndex();
         } else {

src/Storage/Directive/RandomDirectiveHandler.php

@@ -31,6 +31,11 @@ public function __invoke(QueryInterface $query, $value, &$directives): void
 
             return;
         }
+        if ($this->version->getPlatform()['driver_name'] === 'pgsql') {
+            $query->getQueryBuilder()->addSelect('RAND(2) as HIDDEN rand')->orderBy('rand');
+
+            return;
+        }
 
         $query->getQueryBuilder()->addSelect('RAND(0) as HIDDEN rand')->orderBy('rand');
     }

src/Storage/SelectQuery.php

@@ -533,7 +533,9 @@ private function getRegularFieldExpression(Filter $filter, EntityManagerInterfac
 
         $originalLeftExpression = 'content.' . $filter->getKey();
         // LOWER() added to query to enable case insensitive search of JSON  values. Used in conjunction with converting $params of setParameter() to lowercase.
-        $newLeftExpression = JsonHelper::wrapJsonFunction('LOWER(' . $valueAlias . ')', null, $em->getConnection());
+        // BUG SQLSTATE[42883]: Undefined function: 7 ERROR: function lower(jsonb) does not exist
+        // We want to be able to search case-insensitive, database-agnostic, have to think of a good way..
+        $newLeftExpression = JsonHelper::wrapJsonFunction($valueAlias, null, $em->getConnection());
         $valueWhere = $filter->getExpression();
         $valueWhere = str_replace($originalLeftExpression, $newLeftExpression, $valueWhere);
         $expr->add($valueWhere);

templates/helpers/_field_blocks.twig

@@ -43,11 +43,14 @@
 {% block extended_fields %}
 
     {# Special case for 'select' fields: if it's a multiple select, the field is an array. #}
+    {# I dont know what is going on here, but field.selectedIds for pgsql showcase/mr-jean-feeney (and others) #}
+    {# yield field values (strings containing full name!), NOT integer IDs.. #}
+    {# TODO: FIX IT, bypassing for now by selecting on 'value' (string) and not 'id' (int) #}	
     {% if type == "select" and field is not empty %}
         <p><strong>{{ field|label }}: </strong></p>
             <ul>
             {% if field.contentSelect %}
-                {% setcontent selected = field.contentType where {'id': field.selectedIds} %}
+                {% setcontent selected = field.contentType where {'value': field.selectedIds} %}
                 {% for record in selected %}
                 <li><a href="{{ record|link }}">{{ record|title }}</a></li>
                 {% endfor %}

vendor/vaimo/webdriver-binary-downloader/src/Analysers/PlatformAnalyser.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * Copyright © Vaimo Group. All rights reserved.
+ * See LICENSE_VAIMO.txt for license details.
+ */
+namespace Vaimo\WebDriverBinaryDownloader\Analysers;
+
+class PlatformAnalyser implements \Vaimo\WebDriverBinaryDownloader\Interfaces\PlatformAnalyserInterface
+{
+    public function getPlatformCode()
+    {
+        if (stripos(PHP_OS, 'win') === 0) {
+            if (PHP_INT_SIZE === 8) {
+                return self::TYPE_WIN64;
+            }
+
+            return self::TYPE_WIN32;
+        }
+
+        if (stripos(PHP_OS, 'darwin') === 0) {
+            return self::TYPE_MAC64;
+        }
+
+        if (stripos(PHP_OS, 'linux') === 0) {
+            if (PHP_INT_SIZE === 8) {
+                return self::TYPE_LINUX64;
+            }
+
+            return self::TYPE_LINUX32;
+        }
+
+        if (stripos(PHP_OS, 'FreeBSD') === 0) {
+            if (PHP_INT_SIZE === 8) {
+                return self::TYPE_FREEBSD64;
+            }
+
+            return self::TYPE_FREEBSD32;
+        }
+
+
+        throw new \Exception('Platform code detection failed');
+    }
+
+    public function getPlatformName()
+    {
+        $names = array(
+            self::TYPE_LINUX32 => 'Linux 32Bits',
+            self::TYPE_LINUX64 => 'Linux 64Bits',
+            self::TYPE_MAC64 => 'Mac OS X',
+            self::TYPE_WIN32 => 'Windows 32Bits',
+            self::TYPE_WIN64 => 'Windows 64Bits',
+            self::TYPE_FREEBSD64 => 'FreeBSD 64Bits',
+            self::TYPE_FREEBSD32 => 'FreeBSD 32Bits',
+        );
+
+        return $names[$this->getPlatformCode()];
+    }
+}

vendor/vaimo/webdriver-binary-downloader/src/Interfaces/PlatformAnalyserInterface.php

@@ -0,0 +1,27 @@
+<?php
+/**
+ * Copyright © Vaimo Group. All rights reserved.
+ * See LICENSE_VAIMO.txt for license details.
+ */
+namespace Vaimo\WebDriverBinaryDownloader\Interfaces;
+
+interface PlatformAnalyserInterface
+{
+    const TYPE_LINUX32 = 'linux32';
+    const TYPE_LINUX64 = 'linux64';
+    const TYPE_WIN32 = 'win32';
+    const TYPE_WIN64 = 'win64';
+    const TYPE_MAC64 = 'mac64';
+    const TYPE_FREEBSD64 = 'unix64';
+    const TYPE_FREEBSD32 = 'unix32';
+
+    /**
+     * @return string
+     */
+    public function getPlatformCode();
+
+    /**
+     * @return string
+     */
+    public function getPlatformName();
+}