Deparser: Add optional formatting/pretty printing

Rakefile

@@ -66,6 +66,7 @@ task :update_source do
   system("mv #{extdir}/postgres/* #{extdir}/")
   system("rmdir #{extdir}/postgres")
   system("cp -a #{libdir}/pg_query.h #{extdir}/include")
+  system("cp -a #{libdir}/postgres_deparse.h #{extdir}/include")
   # Protobuf definitions
   system("protoc --proto_path=#{libdir}/protobuf --ruby_out=#{File.join(__dir__, 'lib/pg_query')} #{libdir}/protobuf/pg_query.proto")
   system("mkdir -p #{extdir}/include/protobuf")

ext/pg_query/COPYRIGHT

@@ -0,0 +1,23 @@
+PostgreSQL Database Management System
+(formerly known as Postgres, then as Postgres95)
+
+Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+
+Portions Copyright (c) 1994, The Regents of the University of California
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose, without fee, and without a written agreement
+is hereby granted, provided that the above copyright notice and this
+paragraph and the following two paragraphs appear in all copies.
+
+IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR
+DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO
+PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

ext/pg_query/include/pg_query.h

@@ -4,6 +4,8 @@
 #include <stdint.h>
 #include <sys/types.h>
 
+#include "postgres_deparse.h"
+
 typedef struct {
 	char* message; // exception message
 	char* funcname; // source function of exception (e.g. SearchSysCache)
@@ -53,6 +55,12 @@ typedef struct {
   PgQueryError* error;
 } PgQueryDeparseResult;
 
+typedef struct {
+  PostgresDeparseComment **comments;
+  size_t comment_count;
+  PgQueryError* error;
+} PgQueryDeparseCommentsResult;
+
 typedef struct {
   char* plpgsql_funcs;
   PgQueryError* error;
@@ -118,12 +126,15 @@ PgQuerySplitResult pg_query_split_with_scanner(const char *input);
 PgQuerySplitResult pg_query_split_with_parser(const char *input);
 
 PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree);
+PgQueryDeparseResult pg_query_deparse_protobuf_opts(PgQueryProtobuf parse_tree, struct PostgresDeparseOpts opts);
+PgQueryDeparseCommentsResult pg_query_deparse_comments_for_query(const char *query);
 
 void pg_query_free_normalize_result(PgQueryNormalizeResult result);
 void pg_query_free_scan_result(PgQueryScanResult result);
 void pg_query_free_parse_result(PgQueryParseResult result);
 void pg_query_free_split_result(PgQuerySplitResult result);
 void pg_query_free_deparse_result(PgQueryDeparseResult result);
+void pg_query_free_deparse_comments_result(PgQueryDeparseCommentsResult result);
 void pg_query_free_protobuf_parse_result(PgQueryProtobufParseResult result);
 void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result);
 void pg_query_free_fingerprint_result(PgQueryFingerprintResult result);
@@ -133,8 +144,8 @@ void pg_query_exit(void);
 
 // Postgres version information
 #define PG_MAJORVERSION "17"
-#define PG_VERSION "17.4"
-#define PG_VERSION_NUM 170004
+#define PG_VERSION "17.5"
+#define PG_VERSION_NUM 170005
 
 // Deprecated APIs below
 

ext/pg_query/include/postgres/mb/pg_wchar.h

@@ -664,6 +664,8 @@ extern int	pg_valid_server_encoding_id(int encoding);
  */
 extern void pg_encoding_set_invalid(int encoding, char *dst);
 extern int	pg_encoding_mblen(int encoding, const char *mbstr);
+extern int	pg_encoding_mblen_or_incomplete(int encoding, const char *mbstr,
+											size_t remaining);
 extern int	pg_encoding_mblen_bounded(int encoding, const char *mbstr);
 extern int	pg_encoding_dsplen(int encoding, const char *mbstr);
 extern int	pg_encoding_verifymbchar(int encoding, const char *mbstr, int len);

ext/pg_query/include/postgres/nodes/pathnodes.h

@@ -1101,8 +1101,7 @@ typedef struct IndexOptInfo IndexOptInfo;
 #define HAVE_INDEXOPTINFO_TYPEDEF 1
 #endif
 
-struct IndexPath;				/* avoid including pathnodes.h here */
-struct PlannerInfo;				/* avoid including pathnodes.h here */
+struct IndexPath;				/* forward declaration */
 
 struct IndexOptInfo
 {

ext/pg_query/include/postgres/pg_config.h

@@ -592,7 +592,7 @@
 #define PACKAGE_NAME "PostgreSQL"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "PostgreSQL 17.4"
+#define PACKAGE_STRING "PostgreSQL 17.5"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "postgresql"
@@ -601,7 +601,7 @@
 #define PACKAGE_URL "https://www.postgresql.org/"
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "17.4"
+#define PACKAGE_VERSION "17.5"
 
 /* Define to the name of a signed 128-bit integer type. */
 #define PG_INT128_TYPE __int128
@@ -620,7 +620,7 @@
 #define PG_MAJORVERSION_NUM 17
 
 /* PostgreSQL minor version number */
-#define PG_MINORVERSION_NUM 4
+#define PG_MINORVERSION_NUM 5
 
 /* Define to best printf format archetype, usually gnu_printf if available. */
 #define PG_PRINTF_ATTRIBUTE printf
@@ -629,13 +629,13 @@
 #define PG_USE_STDBOOL 1
 
 /* PostgreSQL version as a string */
-#define PG_VERSION "17.4"
+#define PG_VERSION "17.5"
 
 /* PostgreSQL version as a number */
-#define PG_VERSION_NUM 170004
+#define PG_VERSION_NUM 170005
 
 /* A string containing the version number, platform, and C compiler */
-#define PG_VERSION_STR "PostgreSQL 17.4 (libpg_query)"
+#define PG_VERSION_STR "PostgreSQL 17.5 (libpg_query)"
 
 /* Define to 1 to allow profiling output to be saved separately for each
    process. */

ext/pg_query/include/postgres/pg_config_manual.h

@@ -26,7 +26,9 @@
  *
  * Changing this requires an initdb.
  */
+#ifndef NAMEDATALEN
 #define NAMEDATALEN 64
+#endif
 
 /*
  * Maximum number of arguments to a function.

ext/pg_query/include/postgres/replication/reorderbuffer.h

@@ -729,6 +729,10 @@ extern TransactionId *ReorderBufferGetCatalogChangesXacts(ReorderBuffer *rb);
 
 extern void ReorderBufferSetRestartPoint(ReorderBuffer *rb, XLogRecPtr ptr);
 
+extern uint32 ReorderBufferGetInvalidations(ReorderBuffer *rb,
+											TransactionId xid,
+											SharedInvalidationMessage **msgs);
+
 extern void StartupReorderBuffer(void);
 
 #endif

ext/pg_query/include/postgres_deparse.h

@@ -0,0 +1,34 @@
+#ifndef POSTGRES_DEPARSE_H
+#define POSTGRES_DEPARSE_H
+
+#include <stdbool.h>
+#include <sys/types.h>
+
+typedef struct PostgresDeparseComment {
+    int match_location;          // Insert comment before a node, once we find a node whose location field is equal-or-higher than this location
+    int newlines_before_comment; // Insert newlines before inserting the comment (set to non-zero if the source comment was separated from the prior token by at least one newline)
+    int newlines_after_comment;  // Insert newlines after inserting the comment (set to non-zero if the source comment was separated from the next token by at least one newline)
+    char *str;                   // The actual comment string, including comment start/end tokens, and newline characters in comment (if any)
+} PostgresDeparseComment;
+
+typedef struct PostgresDeparseOpts {
+    PostgresDeparseComment **comments;
+    size_t comment_count;
+
+    // Pretty print options
+    bool pretty_print;
+    int indent_size;           // Indentation size (Default 4 spaces)
+    int max_line_length;       // Restricts the line length of certain lists of items (Default 80 characters)
+    bool trailing_newline;     // Whether to add a trailing newline at the end of the output (Default off)
+    bool commas_start_of_line; // Place separating commas at start of line (Default off)
+} PostgresDeparseOpts;
+
+/* Forward declarations to allow referencing the structs in this include file without needing Postgres includes */
+struct StringInfoData;
+typedef struct StringInfoData *StringInfo;
+struct RawStmt;
+
+extern void deparseRawStmt(StringInfo str, struct RawStmt *raw_stmt);
+extern void deparseRawStmtOpts(StringInfo str, struct RawStmt *raw_stmt, PostgresDeparseOpts opts);
+
+#endif

ext/pg_query/pg_query_deparse.c

@@ -4,7 +4,20 @@
 
 #include "postgres_deparse.h"
 
+#include "postgres.h"
+#include "lib/stringinfo.h"
+#include "nodes/parsenodes.h"
+
+#include "protobuf/pg_query.pb-c.h"
+
 PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree)
+{
+	PostgresDeparseOpts opts;
+	MemSet(&opts, 0, sizeof(PostgresDeparseOpts));
+	return pg_query_deparse_protobuf_opts(parse_tree, opts);
+}
+
+PgQueryDeparseResult pg_query_deparse_protobuf_opts(PgQueryProtobuf parse_tree, PostgresDeparseOpts opts)
 {
 	PgQueryDeparseResult result = {0};
 	StringInfoData str;
@@ -21,7 +34,7 @@ PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree)
 		initStringInfo(&str);
 
 		foreach(lc, stmts) {
-			deparseRawStmt(&str, castNode(RawStmt, lfirst(lc)));
+			deparseRawStmtOpts(&str, castNode(RawStmt, lfirst(lc)), opts);
 			if (lnext(stmts, lc))
 				appendStringInfoString(&str, "; ");
 		}
@@ -62,3 +75,103 @@ void pg_query_free_deparse_result(PgQueryDeparseResult result)
 
 	free(result.query);
 }
+
+PgQueryDeparseCommentsResult pg_query_deparse_comments_for_query(const char *query)
+{
+	PgQueryDeparseCommentsResult result = {0};
+	PgQueryScanResult scan_result_raw = pg_query_scan(query);
+	if (scan_result_raw.error)
+	{
+		result.error = scan_result_raw.error;
+		return result;
+	}
+
+    PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack(NULL, scan_result_raw.pbuf.len, (void*) scan_result_raw.pbuf.data);
+	bool prior_token_was_comment = false;
+	int32_t prior_non_comment_end = 0;
+	int32_t prior_token_end = 0;
+
+	result.comment_count = 0;
+	for (int i = 0; i < scan_result->n_tokens; i++)
+	{
+        PgQuery__ScanToken *token = scan_result->tokens[i];
+		if (token->token == PG_QUERY__TOKEN__SQL_COMMENT || token->token == PG_QUERY__TOKEN__C_COMMENT)
+			result.comment_count++;
+	}
+
+	result.comments = malloc(result.comment_count * sizeof(PostgresDeparseComment*));
+	size_t comment_idx = 0;
+    for (int i = 0; i < scan_result->n_tokens; i++)
+	{
+        PgQuery__ScanToken *token = scan_result->tokens[i];
+		if (token->token == PG_QUERY__TOKEN__SQL_COMMENT || token->token == PG_QUERY__TOKEN__C_COMMENT)
+		{
+			size_t token_len = token->end - token->start;
+			PostgresDeparseComment* comment = malloc(sizeof(PostgresDeparseComment));
+
+			/*
+			 * Set to the end of the prior non-comment token.
+			 *
+			 * We could alternatively set the match location to the start of the next
+			 * non-comment token instead (which would be more in line with emitting
+			 * that happens before the node once the location is matched).
+			 *
+			 * However, that doesn't work well in practice, specifically. RawStmt start
+			 * locations are 0, even when there are comments at the start of the statement.
+			 */
+			comment->match_location = prior_non_comment_end;
+
+			comment->newlines_before_comment = 0;
+			comment->newlines_after_comment  = 0;
+
+			// Emit newlines before the comment if prior token was not a comment (otherwise that comment would have emitted them)
+			if (!prior_token_was_comment)
+			{
+				for (int j = prior_token_end; j < token->start; j++)
+				{
+					if (query[j] == '\n')
+						comment->newlines_before_comment++;
+				}
+			}
+
+			if (i < scan_result->n_tokens - 1)
+			{
+				for (int j = token->end; j < scan_result->tokens[i + 1]->start; j++)
+				{
+					if (query[j] == '\n')
+						comment->newlines_after_comment++;
+				}
+			}
+
+			comment->str = malloc(token_len + 1);
+			memcpy(comment->str, &(query[token->start]), token_len);
+			comment->str[token_len] = '\0';
+
+			result.comments[comment_idx] = comment;
+			comment_idx++;
+			prior_token_was_comment = true;
+		}
+		else
+		{
+			prior_non_comment_end = token->end;
+			prior_token_was_comment = false;
+		}
+		prior_token_end = token->end;
+    }
+
+    pg_query__scan_result__free_unpacked(scan_result, NULL);
+    pg_query_free_scan_result(scan_result_raw);
+
+	return result;
+}
+
+void pg_query_free_deparse_comments_result(PgQueryDeparseCommentsResult result)
+{
+	for (int i = 0; i < result.comment_count; i++)
+	{
+		free(result.comments[i]->str);
+		free(result.comments[i]);
+	}
+	if (result.comments != NULL)
+		free(result.comments);
+}