Skip to content

Commit a6fe5ea

Browse files
authored
Merge pull request #4571 from neondatabase/releases/2023-06-27
Release 2023-06-27
2 parents cd17053 + 05b0aed commit a6fe5ea

File tree

44 files changed

+979
-417
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+979
-417
lines changed

.github/workflows/benchmarking.yml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,8 @@ jobs:
180180
image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
181181
options: --init
182182

183-
timeout-minutes: 360 # 6h
183+
# Increase timeout to 8h, default timeout is 6h
184+
timeout-minutes: 480
184185

185186
steps:
186187
- uses: actions/checkout@v3
@@ -321,8 +322,6 @@ jobs:
321322
image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
322323
options: --init
323324

324-
timeout-minutes: 360 # 6h
325-
326325
steps:
327326
- uses: actions/checkout@v3
328327

@@ -414,8 +413,6 @@ jobs:
414413
image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
415414
options: --init
416415

417-
timeout-minutes: 360 # 6h
418-
419416
steps:
420417
- uses: actions/checkout@v3
421418

@@ -501,8 +498,6 @@ jobs:
501498
image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
502499
options: --init
503500

504-
timeout-minutes: 360 # 6h
505-
506501
steps:
507502
- uses: actions/checkout@v3
508503

.github/workflows/build_and_test.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,7 @@ jobs:
659659
--cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache
660660
--context .
661661
--build-arg GIT_VERSION=${{ github.sha }}
662+
--build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}}
662663
--build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
663664
--dockerfile Dockerfile.compute-tools
664665
--destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}}
@@ -716,6 +717,7 @@ jobs:
716717
--context .
717718
--build-arg GIT_VERSION=${{ github.sha }}
718719
--build-arg PG_VERSION=${{ matrix.version }}
720+
--build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}}
719721
--build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
720722
--dockerfile Dockerfile.compute-node
721723
--destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
@@ -736,7 +738,7 @@ jobs:
736738
run:
737739
shell: sh -eu {0}
738740
env:
739-
VM_BUILDER_VERSION: v0.8.0
741+
VM_BUILDER_VERSION: v0.11.0
740742

741743
steps:
742744
- name: Checkout
@@ -914,7 +916,7 @@ jobs:
914916
exit 1
915917
fi
916918
917-
- name: Create tag "release-${{ needs.tag.outputs.build-tag }}"
919+
- name: Create git tag
918920
if: github.ref_name == 'release'
919921
uses: actions/github-script@v6
920922
with:
@@ -924,7 +926,7 @@ jobs:
924926
github.rest.git.createRef({
925927
owner: context.repo.owner,
926928
repo: context.repo.repo,
927-
ref: "refs/tags/release-${{ needs.tag.outputs.build-tag }}",
929+
ref: "refs/tags/${{ needs.tag.outputs.build-tag }}",
928930
sha: context.sha,
929931
})
930932

Dockerfile.compute-node

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ ARG PG_VERSION
22
ARG REPOSITORY=neondatabase
33
ARG IMAGE=rust
44
ARG TAG=pinned
5+
ARG BUILD_TAG
56

67
#########################################################################################
78
#
@@ -480,6 +481,40 @@ RUN wget https://github.com/rdkit/rdkit/archive/refs/tags/Release_2023_03_1.tar.
480481
make -j $(getconf _NPROCESSORS_ONLN) install && \
481482
echo 'trusted = true' >> /usr/local/pgsql/share/extension/rdkit.control
482483

484+
#########################################################################################
485+
#
486+
# Layer "pg-uuidv7-pg-build"
487+
# compile pg_uuidv7 extension
488+
#
489+
#########################################################################################
490+
FROM build-deps AS pg-uuidv7-pg-build
491+
COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
492+
493+
ENV PATH "/usr/local/pgsql/bin/:$PATH"
494+
RUN wget https://github.com/fboulnois/pg_uuidv7/archive/refs/tags/v1.0.1.tar.gz -O pg_uuidv7.tar.gz && \
495+
echo "0d0759ab01b7fb23851ecffb0bce27822e1868a4a5819bfd276101c716637a7a pg_uuidv7.tar.gz" | sha256sum --check && \
496+
mkdir pg_uuidv7-src && cd pg_uuidv7-src && tar xvzf ../pg_uuidv7.tar.gz --strip-components=1 -C . && \
497+
make -j $(getconf _NPROCESSORS_ONLN) && \
498+
make -j $(getconf _NPROCESSORS_ONLN) install && \
499+
echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_uuidv7.control
500+
501+
#########################################################################################
502+
#
503+
# Layer "pg-roaringbitmap-pg-build"
504+
# compile pg_roaringbitmap extension
505+
#
506+
#########################################################################################
507+
FROM build-deps AS pg-roaringbitmap-pg-build
508+
COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
509+
510+
ENV PATH "/usr/local/pgsql/bin/:$PATH"
511+
RUN wget https://github.com/ChenHuajun/pg_roaringbitmap/archive/refs/tags/v0.5.4.tar.gz -O pg_roaringbitmap.tar.gz && \
512+
echo "b75201efcb1c2d1b014ec4ae6a22769cc7a224e6e406a587f5784a37b6b5a2aa pg_roaringbitmap.tar.gz" | sha256sum --check && \
513+
mkdir pg_roaringbitmap-src && cd pg_roaringbitmap-src && tar xvzf ../pg_roaringbitmap.tar.gz --strip-components=1 -C . && \
514+
make -j $(getconf _NPROCESSORS_ONLN) && \
515+
make -j $(getconf _NPROCESSORS_ONLN) install && \
516+
echo 'trusted = true' >> /usr/local/pgsql/share/extension/roaringbitmap.control
517+
483518
#########################################################################################
484519
#
485520
# Layer "rust extensions"
@@ -613,6 +648,8 @@ COPY --from=kq-imcx-pg-build /usr/local/pgsql/ /usr/local/pgsql/
613648
COPY --from=pg-cron-pg-build /usr/local/pgsql/ /usr/local/pgsql/
614649
COPY --from=pg-pgx-ulid-build /usr/local/pgsql/ /usr/local/pgsql/
615650
COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/
651+
COPY --from=pg-uuidv7-pg-build /usr/local/pgsql/ /usr/local/pgsql/
652+
COPY --from=pg-roaringbitmap-pg-build /usr/local/pgsql/ /usr/local/pgsql/
616653
COPY pgxn/ pgxn/
617654

618655
RUN make -j $(getconf _NPROCESSORS_ONLN) \
@@ -634,6 +671,9 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \
634671
#
635672
#########################################################################################
636673
FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools
674+
ARG BUILD_TAG
675+
ENV BUILD_TAG=$BUILD_TAG
676+
637677
USER nonroot
638678
# Copy entire project to get Cargo.* files with proper dependencies for the whole project
639679
COPY --chown=nonroot . .

Dockerfile.compute-tools

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
ARG REPOSITORY=neondatabase
44
ARG IMAGE=rust
55
ARG TAG=pinned
6+
ARG BUILD_TAG
67

78
FROM $REPOSITORY/$IMAGE:$TAG AS rust-build
89
WORKDIR /home/nonroot
@@ -16,6 +17,8 @@ ENV CACHEPOT_S3_KEY_PREFIX=cachepot
1617
ARG CACHEPOT_BUCKET=neon-github-dev
1718
#ARG AWS_ACCESS_KEY_ID
1819
#ARG AWS_SECRET_ACCESS_KEY
20+
ARG BUILD_TAG
21+
ENV BUILD_TAG=$BUILD_TAG
1922

2023
COPY . .
2124

compute_tools/src/bin/compute_ctl.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,15 @@ use compute_tools::monitor::launch_monitor;
5454
use compute_tools::params::*;
5555
use compute_tools::spec::*;
5656

57+
const BUILD_TAG_DEFAULT: &str = "local";
58+
5759
fn main() -> Result<()> {
5860
init_tracing_and_logging(DEFAULT_LOG_LEVEL)?;
5961

62+
let build_tag = option_env!("BUILD_TAG").unwrap_or(BUILD_TAG_DEFAULT);
63+
64+
info!("build_tag: {build_tag}");
65+
6066
let matches = cli().get_matches();
6167

6268
let http_port = *matches

compute_tools/src/compute.rs

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,84 @@ impl TryFrom<ComputeSpec> for ParsedSpec {
133133
}
134134
}
135135

136+
/// Create special neon_superuser role, that's a slightly nerfed version of a real superuser
137+
/// that we give to customers
138+
fn create_neon_superuser(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
139+
let roles = spec
140+
.cluster
141+
.roles
142+
.iter()
143+
.map(|r| format!("'{}'", escape_literal(&r.name)))
144+
.collect::<Vec<_>>();
145+
146+
let dbs = spec
147+
.cluster
148+
.databases
149+
.iter()
150+
.map(|db| format!("'{}'", escape_literal(&db.name)))
151+
.collect::<Vec<_>>();
152+
153+
let roles_decl = if roles.is_empty() {
154+
String::from("roles text[] := NULL;")
155+
} else {
156+
format!(
157+
r#"
158+
roles text[] := ARRAY(SELECT rolname
159+
FROM pg_catalog.pg_roles
160+
WHERE rolname IN ({}));"#,
161+
roles.join(", ")
162+
)
163+
};
164+
165+
let database_decl = if dbs.is_empty() {
166+
String::from("dbs text[] := NULL;")
167+
} else {
168+
format!(
169+
r#"
170+
dbs text[] := ARRAY(SELECT datname
171+
FROM pg_catalog.pg_database
172+
WHERE datname IN ({}));"#,
173+
dbs.join(", ")
174+
)
175+
};
176+
177+
// ALL PRIVILEGES grants CREATE, CONNECT, and TEMPORARY on all databases
178+
// (see https://www.postgresql.org/docs/current/ddl-priv.html)
179+
let query = format!(
180+
r#"
181+
DO $$
182+
DECLARE
183+
r text;
184+
{}
185+
{}
186+
BEGIN
187+
IF NOT EXISTS (
188+
SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser')
189+
THEN
190+
CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN IN ROLE pg_read_all_data, pg_write_all_data;
191+
IF array_length(roles, 1) IS NOT NULL THEN
192+
EXECUTE format('GRANT neon_superuser TO %s',
193+
array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(roles) as x), ', '));
194+
FOREACH r IN ARRAY roles LOOP
195+
EXECUTE format('ALTER ROLE %s CREATEROLE CREATEDB', quote_ident(r));
196+
END LOOP;
197+
END IF;
198+
IF array_length(dbs, 1) IS NOT NULL THEN
199+
EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %s TO neon_superuser',
200+
array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(dbs) as x), ', '));
201+
END IF;
202+
END IF;
203+
END
204+
$$;"#,
205+
roles_decl, database_decl,
206+
);
207+
info!("Neon superuser created:\n{}", &query);
208+
client
209+
.simple_query(&query)
210+
.map_err(|e| anyhow::anyhow!(e).context(query))?;
211+
Ok(())
212+
}
213+
136214
impl ComputeNode {
137215
pub fn set_status(&self, status: ComputeStatus) {
138216
let mut state = self.state.lock().unwrap();
@@ -347,6 +425,8 @@ impl ComputeNode {
347425
.map_err(|_| anyhow::anyhow!("invalid connstr"))?;
348426

349427
let mut client = Client::connect(zenith_admin_connstr.as_str(), NoTls)?;
428+
// Disable forwarding so that users don't get a cloud_admin role
429+
client.simple_query("SET neon.forward_ddl = false")?;
350430
client.simple_query("CREATE USER cloud_admin WITH SUPERUSER")?;
351431
client.simple_query("GRANT zenith_admin TO cloud_admin")?;
352432
drop(client);
@@ -357,14 +437,16 @@ impl ComputeNode {
357437
Ok(client) => client,
358438
};
359439

360-
// Proceed with post-startup configuration. Note, that order of operations is important.
361440
// Disable DDL forwarding because control plane already knows about these roles/databases.
362441
client.simple_query("SET neon.forward_ddl = false")?;
442+
443+
// Proceed with post-startup configuration. Note, that order of operations is important.
363444
let spec = &compute_state.pspec.as_ref().expect("spec must be set").spec;
445+
create_neon_superuser(spec, &mut client)?;
364446
handle_roles(spec, &mut client)?;
365447
handle_databases(spec, &mut client)?;
366448
handle_role_deletions(spec, self.connstr.as_str(), &mut client)?;
367-
handle_grants(spec, self.connstr.as_str(), &mut client)?;
449+
handle_grants(spec, self.connstr.as_str())?;
368450
handle_extensions(spec, &mut client)?;
369451

370452
// 'Close' connection
@@ -402,7 +484,7 @@ impl ComputeNode {
402484
handle_roles(&spec, &mut client)?;
403485
handle_databases(&spec, &mut client)?;
404486
handle_role_deletions(&spec, self.connstr.as_str(), &mut client)?;
405-
handle_grants(&spec, self.connstr.as_str(), &mut client)?;
487+
handle_grants(&spec, self.connstr.as_str())?;
406488
handle_extensions(&spec, &mut client)?;
407489
}
408490

compute_tools/src/pg_helpers.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use compute_api::spec::{Database, GenericOption, GenericOptions, PgIdent, Role};
1717
const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds
1818

1919
/// Escape a string for including it in a SQL literal
20-
fn escape_literal(s: &str) -> String {
20+
pub fn escape_literal(s: &str) -> String {
2121
s.replace('\'', "''").replace('\\', "\\\\")
2222
}
2323

compute_tools/src/spec.rs

Lines changed: 10 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -269,17 +269,13 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
269269
xact.execute(query.as_str(), &[])?;
270270
}
271271
RoleAction::Create => {
272-
let mut query: String = format!("CREATE ROLE {} ", name.pg_quote());
272+
let mut query: String = format!(
273+
"CREATE ROLE {} CREATEROLE CREATEDB IN ROLE neon_superuser",
274+
name.pg_quote()
275+
);
273276
info!("role create query: '{}'", &query);
274277
query.push_str(&role.to_pg_options());
275278
xact.execute(query.as_str(), &[])?;
276-
277-
let grant_query = format!(
278-
"GRANT pg_read_all_data, pg_write_all_data TO {}",
279-
name.pg_quote()
280-
);
281-
xact.execute(grant_query.as_str(), &[])?;
282-
info!("role grant query: '{}'", &grant_query);
283279
}
284280
}
285281

@@ -476,6 +472,11 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
476472
query.push_str(&db.to_pg_options());
477473
let _guard = info_span!("executing", query).entered();
478474
client.execute(query.as_str(), &[])?;
475+
let grant_query: String = format!(
476+
"GRANT ALL PRIVILEGES ON DATABASE {} TO neon_superuser",
477+
name.pg_quote()
478+
);
479+
client.execute(grant_query.as_str(), &[])?;
479480
}
480481
};
481482

@@ -495,35 +496,9 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
495496
/// Grant CREATE ON DATABASE to the database owner and do some other alters and grants
496497
/// to allow users creating trusted extensions and re-creating `public` schema, for example.
497498
#[instrument(skip_all)]
498-
pub fn handle_grants(spec: &ComputeSpec, connstr: &str, client: &mut Client) -> Result<()> {
499+
pub fn handle_grants(spec: &ComputeSpec, connstr: &str) -> Result<()> {
499500
info!("cluster spec grants:");
500501

501-
// We now have a separate `web_access` role to connect to the database
502-
// via the web interface and proxy link auth. And also we grant a
503-
// read / write all data privilege to every role. So also grant
504-
// create to everyone.
505-
// XXX: later we should stop messing with Postgres ACL in such horrible
506-
// ways.
507-
let roles = spec
508-
.cluster
509-
.roles
510-
.iter()
511-
.map(|r| r.name.pg_quote())
512-
.collect::<Vec<_>>();
513-
514-
for db in &spec.cluster.databases {
515-
let dbname = &db.name;
516-
517-
let query: String = format!(
518-
"GRANT CREATE ON DATABASE {} TO {}",
519-
dbname.pg_quote(),
520-
roles.join(", ")
521-
);
522-
info!("grant query {}", &query);
523-
524-
client.execute(query.as_str(), &[])?;
525-
}
526-
527502
// Do some per-database access adjustments. We'd better do this at db creation time,
528503
// but CREATE DATABASE isn't transactional. So we cannot create db + do some grants
529504
// atomically.

0 commit comments

Comments
 (0)