From fa623894828da97f18fcbfabe759097dfd718596 Mon Sep 17 00:00:00 2001 From: takuya kodama Date: Mon, 1 Jul 2024 10:07:50 +0800 Subject: [PATCH 01/29] packages almalinux: add `--platform linux/arm64` when building arm64's image (#1807) ref: https://github.com/groonga/groonga/issues/1806 This change solves the following error on CI. Because Docker build process does not automatically detect the correct platform for the image. As a result, the platform information defaults to the host platform. So in this case, platform infromation wasn't as a arm64. To resolve this issue, we need to specify the platform explicitly in the Docker build command. ref: https://docs.docker.com/build/building/multi-platform/ Here is the error log: https://github.com/groonga/groonga/actions/runs/9705591558/job/26788067581#step:9:42 ``` ------ #3 [internal] load metadata for docker.io/arm64v8/almalinux:8 #3 ERROR: no match for platform in manifest: not found ------ Dockerfile:2 -------------------- 1 | ARG FROM=almalinux:8 2 | >>> FROM ${FROM} 3 | 4 | ARG DEBUG -------------------- ERROR: failed to solve: arm64v8/almalinux:8: failed to resolve source metadata for docker.io/arm64v8/almalinux:8: no match for platform in manifest: not found rake aborted!#1 [internal] load build definition from Dockerfile ``` --------- Co-authored-by: Horimoto Yasuhiro Co-authored-by: Sutou Kouhei --- .github/workflows/package.yml | 9 +++++++++ packages/yum/almalinux-8-aarch64/from | 2 +- packages/yum/almalinux-9-aarch64/from | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 2ed2c3c0d4..7b74ad4158 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -327,7 +327,16 @@ jobs: # Test - name: Test run: | + case "${{ matrix.test-docker-image }}" in + arm64v8/*) + platform=linux/arm64 + ;; + *) + platform=linux/amd64 + ;; + esac docker run \ + --platform ${platform} \ --rm \ --volume ${PWD}:/groonga:ro \ ${{ matrix.test-docker-image }} \ diff --git a/packages/yum/almalinux-8-aarch64/from b/packages/yum/almalinux-8-aarch64/from index dcefc318ee..dc6d522a92 100644 --- a/packages/yum/almalinux-8-aarch64/from +++ b/packages/yum/almalinux-8-aarch64/from @@ -1 +1 @@ -arm64v8/almalinux:8 +--platform=linux/arm64 arm64v8/almalinux:8 diff --git a/packages/yum/almalinux-9-aarch64/from b/packages/yum/almalinux-9-aarch64/from index 4873d2ca10..9154373c27 100644 --- a/packages/yum/almalinux-9-aarch64/from +++ b/packages/yum/almalinux-9-aarch64/from @@ -1 +1 @@ -arm64v8/almalinux:9 +--platform=linux/arm64 arm64v8/almalinux:9 From f69b682612e99796ff4cd445a8486d10052d3f76 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 13:15:44 +0900 Subject: [PATCH 02/29] GRN_N_HOOK_ENTRIES: move from internal code It should be defines at close to grn_hook_entry. --- include/groonga/groonga.h | 2 ++ lib/db.c | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/groonga/groonga.h b/include/groonga/groonga.h index c29a3465ba..16fb75881f 100644 --- a/include/groonga/groonga.h +++ b/include/groonga/groonga.h @@ -1046,6 +1046,8 @@ typedef enum { GRN_HOOK_SELECT } grn_hook_entry; +#define GRN_N_HOOK_ENTRIES 5 /* (GRN_HOOK_SELECT + 1) */ + GRN_API grn_rc grn_obj_add_hook(grn_ctx *ctx, grn_obj *obj, diff --git a/lib/db.c b/lib/db.c index 5af79f03a6..2c9d8b0452 100644 --- a/lib/db.c +++ b/lib/db.c @@ -3548,7 +3548,7 @@ grn_table_cursor_close(grn_ctx *ctx, grn_table_cursor *tc) } /* grn_hook_entry entry; - for (entry = 0; entry < N_HOOK_ENTRIES; entry++) { + for (entry = 0; entry < GRN_N_HOOK_ENTRIES; entry++) { grn_hook_free(ctx, DB_OBJ(tc)->hooks[entry]); } */ @@ -8092,14 +8092,13 @@ delete_source_hook(grn_ctx *ctx, grn_obj *obj) grn_obj_close(ctx, &data); } -#define N_HOOK_ENTRIES 5 grn_rc grn_hook_pack(grn_ctx *ctx, grn_db_obj *obj, grn_obj *buf) { grn_rc rc; grn_hook_entry e; - for (e = 0; e < N_HOOK_ENTRIES; e++) { + for (e = 0; e < GRN_N_HOOK_ENTRIES; e++) { grn_hook *hooks; for (hooks = obj->hooks[e]; hooks; hooks = hooks->next) { grn_id id = hooks->proc ? hooks->proc->obj.id : 0; @@ -8132,7 +8131,7 @@ grn_hook_unpack(grn_ctx *ctx, { grn_hook_entry e; const uint8_t *p = (uint8_t *)buf, *pe = p + buf_size; - for (e = 0; e < N_HOOK_ENTRIES; e++) { + for (e = 0; e < GRN_N_HOOK_ENTRIES; e++) { grn_hook *new, **last = &obj->hooks[e]; for (;;) { grn_id id; @@ -11233,7 +11232,7 @@ grn_db_obj_init(grn_ctx *ctx, grn_obj *db, grn_id id, grn_db_obj *obj) obj->source_size = 0; { grn_hook_entry entry; - for (entry = 0; entry < N_HOOK_ENTRIES; entry++) { + for (entry = 0; entry < GRN_N_HOOK_ENTRIES; entry++) { obj->hooks[entry] = NULL; } } @@ -12726,7 +12725,7 @@ grn_obj_close(grn_ctx *ctx, grn_obj *obj) if (DB_OBJ(obj)->source) { GRN_FREE(DB_OBJ(obj)->source); } - for (entry = 0; entry < N_HOOK_ENTRIES; entry++) { + for (entry = 0; entry < GRN_N_HOOK_ENTRIES; entry++) { grn_hook_free(ctx, DB_OBJ(obj)->hooks[entry]); } if (id & GRN_OBJ_TMP_OBJECT) { From 29fafff615e642a7ee6e8dc8f30f01aacb3bb8d8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 13:17:31 +0900 Subject: [PATCH 03/29] test table_remove/broken: reconstruct --- .../broken/{column.expected => data_column.expected} | 0 .../suite/table_remove/broken/{column.test => data_column.test} | 1 + .../column.expected | 0 .../dangling_reference => refer_broken_index_column}/column.test | 1 + .../table.expected | 0 .../dangling_reference => refer_broken_index_column}/table.test | 1 + 6 files changed, 3 insertions(+) rename test/command/suite/table_remove/broken/{column.expected => data_column.expected} (100%) rename test/command/suite/table_remove/broken/{column.test => data_column.test} (96%) rename test/command/suite/table_remove/broken/{index/dangling_reference => refer_broken_index_column}/column.expected (100%) rename test/command/suite/table_remove/broken/{index/dangling_reference => refer_broken_index_column}/column.test (96%) rename test/command/suite/table_remove/broken/{index/dangling_reference => refer_broken_index_column}/table.expected (100%) rename test/command/suite/table_remove/broken/{index/dangling_reference => refer_broken_index_column}/table.test (96%) diff --git a/test/command/suite/table_remove/broken/column.expected b/test/command/suite/table_remove/broken/data_column.expected similarity index 100% rename from test/command/suite/table_remove/broken/column.expected rename to test/command/suite/table_remove/broken/data_column.expected diff --git a/test/command/suite/table_remove/broken/column.test b/test/command/suite/table_remove/broken/data_column.test similarity index 96% rename from test/command/suite/table_remove/broken/column.test rename to test/command/suite/table_remove/broken/data_column.test index 395314599d..5b2eadde17 100644 --- a/test/command/suite/table_remove/broken/column.test +++ b/test/command/suite/table_remove/broken/data_column.test @@ -7,6 +7,7 @@ column_create Users zzz COLUMN_SCALAR Int32 thread_limit 1 #@enable-logging database_unmap +# Users.name #@copy-path fixture/object_remove/too_small.data #{db_path}.0000102 table_remove Users --ensure yes diff --git a/test/command/suite/table_remove/broken/index/dangling_reference/column.expected b/test/command/suite/table_remove/broken/refer_broken_index_column/column.expected similarity index 100% rename from test/command/suite/table_remove/broken/index/dangling_reference/column.expected rename to test/command/suite/table_remove/broken/refer_broken_index_column/column.expected diff --git a/test/command/suite/table_remove/broken/index/dangling_reference/column.test b/test/command/suite/table_remove/broken/refer_broken_index_column/column.test similarity index 96% rename from test/command/suite/table_remove/broken/index/dangling_reference/column.test rename to test/command/suite/table_remove/broken/refer_broken_index_column/column.test index a05e74d1a7..657a38fd06 100644 --- a/test/command/suite/table_remove/broken/index/dangling_reference/column.test +++ b/test/command/suite/table_remove/broken/refer_broken_index_column/column.test @@ -9,6 +9,7 @@ column_create Terms index COLUMN_INDEX|WITH_POSITION Users name thread_limit 1 #@enable-logging database_unmap +# Terms.index #@copy-path fixture/object_remove/too_small.data #{db_path}.0000103 table_remove Users --ensure yes diff --git a/test/command/suite/table_remove/broken/index/dangling_reference/table.expected b/test/command/suite/table_remove/broken/refer_broken_index_column/table.expected similarity index 100% rename from test/command/suite/table_remove/broken/index/dangling_reference/table.expected rename to test/command/suite/table_remove/broken/refer_broken_index_column/table.expected diff --git a/test/command/suite/table_remove/broken/index/dangling_reference/table.test b/test/command/suite/table_remove/broken/refer_broken_index_column/table.test similarity index 96% rename from test/command/suite/table_remove/broken/index/dangling_reference/table.test rename to test/command/suite/table_remove/broken/refer_broken_index_column/table.test index c0d61bd17c..90764776db 100644 --- a/test/command/suite/table_remove/broken/index/dangling_reference/table.test +++ b/test/command/suite/table_remove/broken/refer_broken_index_column/table.test @@ -8,6 +8,7 @@ column_create Terms index COLUMN_INDEX|WITH_POSITION Users _key thread_limit 1 #@enable-logging database_unmap +# Terms.index #@copy-path fixture/object_remove/too_small.data #{db_path}.0000102 table_remove Users --ensure yes From 42ce2b5a9f21d851490902bdaa65144532918d1c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 15:19:53 +0900 Subject: [PATCH 04/29] grn_ctx_remove*: add support for removing hooks for broken index --- lib/db.c | 113 ++++++++++++++++++ .../table_remove/broken/index_column.expected | 23 ++++ .../table_remove/broken/index_column.test | 21 ++++ 3 files changed, 157 insertions(+) create mode 100644 test/command/suite/table_remove/broken/index_column.expected create mode 100644 test/command/suite/table_remove/broken/index_column.test diff --git a/lib/db.c b/lib/db.c index 2c9d8b0452..529f157bd6 100644 --- a/lib/db.c +++ b/lib/db.c @@ -8092,6 +8092,108 @@ delete_source_hook(grn_ctx *ctx, grn_obj *obj) grn_obj_close(ctx, &data); } +/* This should be used only when the target object can't be opened. If + * the target object can be opened, use delete_source_hook() instead. */ +/* static void */ +static void +delete_source_hook_full_scan(grn_ctx *ctx, grn_id target_id, uint32_t flags) +{ + bool is_close_opened_object_mode = false; + if (grn_thread_get_limit() == 1) { + is_close_opened_object_mode = true; + } + + GRN_DB_EACH_SPEC_BEGIN(ctx, cursor, id, spec) + { + if (id == target_id) { + continue; + } + + bool may_have_source = false; + switch (spec->header.type) { + case GRN_TABLE_HASH_KEY: + case GRN_TABLE_PAT_KEY: + case GRN_TABLE_DAT_KEY: + case GRN_TABLE_NO_KEY: + case GRN_COLUMN_VAR_SIZE: + case GRN_COLUMN_FIX_SIZE: + may_have_source = true; + break; + default: + break; + } + if (!may_have_source) { + continue; + } + + if (is_close_opened_object_mode) { + grn_ctx_push_temporary_open_space(ctx); + } + + /* TODO: We should not access to internal variable in + * GRN_DB_EACH_SPEC_BEGIN(). */ + const char *raw_hooks; + uint32_t size = grn_vector_get_element(ctx, + &decoded_spec, + GRN_SERIALIZED_SPEC_INDEX_HOOK, + &raw_hooks, + NULL, + NULL); + + grn_obj *source = NULL; + /* TODO: Unify with grn_hook_unpack() */ + grn_hook_entry entry; + const uint8_t *current = (const uint8_t *)raw_hooks; + const uint8_t *end = current + size; + for (entry = 0; entry < GRN_N_HOOK_ENTRIES && ctx->rc == GRN_SUCCESS; + entry++) { + int i; + for (i = 0; true; i++) { + uint32_t hook_proc_id_plus_one; + GRN_B_DEC(hook_proc_id_plus_one, current); + if (hook_proc_id_plus_one == 0) { + break; + } + grn_id hook_proc_id = hook_proc_id_plus_one - 1; + if (current >= end) { + break; + } + uint32_t hold_size; + GRN_B_DEC(hold_size, current); + if (current >= end) { + break; + } + if (current + hold_size >= end) { + break; + } + if (hook_proc_id == GRN_ID_NIL) { + grn_obj_default_set_value_hook_data *data = + (grn_obj_default_set_value_hook_data *)current; + if (data->target == target_id) { + if (!source) { + source = grn_ctx_at(ctx, id); + if (!source) { + break; + } + } + grn_obj_delete_hook(ctx, source, entry, i); + if (ctx->rc != GRN_SUCCESS) { + break; + } + } + } + current += hold_size; + } + } + + if (is_close_opened_object_mode) { + grn_ctx_pop_temporary_open_space(ctx); + } + } + GRN_DB_EACH_SPEC_END(ctx, cursor); + + return; +} grn_rc grn_hook_pack(grn_ctx *ctx, grn_db_obj *obj, grn_obj *buf) @@ -10715,6 +10817,17 @@ grn_ctx_remove_internal( } } } else if (grn_obj_type_is_column(type)) { + switch (type) { + case GRN_COLUMN_VAR_SIZE: + case GRN_COLUMN_INDEX: + delete_source_hook_full_scan(ctx, id, flags); + if (ctx->rc != GRN_SUCCESS) { + return ctx->rc; + } + break; + default: + break; + } grn_rc rc = remove_index_full_scan(ctx, id, flags); if (rc != GRN_SUCCESS) { return rc; diff --git a/test/command/suite/table_remove/broken/index_column.expected b/test/command/suite/table_remove/broken/index_column.expected new file mode 100644 index 0000000000..71178df235 --- /dev/null +++ b/test/command/suite/table_remove/broken/index_column.expected @@ -0,0 +1,23 @@ +table_create Users TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Users name COLUMN_SCALAR ShortText +[[0,0.0,0.0],true] +table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto +[[0,0.0,0.0],true] +column_create Terms index COLUMN_INDEX|WITH_POSITION Users name +[[0,0.0,0.0],true] +database_unmap +[[0,0.0,0.0],true] +table_remove Terms --ensure yes +[[0,0.0,0.0],true] +#|e| [io][open] file size is too small: <8>(required: >= 64): +#|e| grn_ctx_at: failed to open object: <259>():<72>() +#|e| [table][remove][columns][Terms] column is broken: (259): [io][open] file size is too small: <8>(required: >= 64): +#|e| [io][open] file size is too small: <8>(required: >= 64): +#|e| grn_ctx_at: failed to open object: <259>():<72>() +dump +table_create Users TABLE_NO_KEY +column_create Users name COLUMN_SCALAR ShortText +table_remove Users +[[0,0.0,0.0],true] +dump diff --git a/test/command/suite/table_remove/broken/index_column.test b/test/command/suite/table_remove/broken/index_column.test new file mode 100644 index 0000000000..108c1ba53d --- /dev/null +++ b/test/command/suite/table_remove/broken/index_column.test @@ -0,0 +1,21 @@ +table_create Users TABLE_NO_KEY +column_create Users name COLUMN_SCALAR ShortText +table_create Terms TABLE_PAT_KEY ShortText \ + --default_tokenizer TokenBigram \ + --normalizer NormalizerAuto +column_create Terms index COLUMN_INDEX|WITH_POSITION Users name + +#@disable-logging +thread_limit 1 +#@enable-logging +database_unmap +# Terms.index +#@copy-path fixture/object_remove/too_small.data #{db_path}.0000103 + +table_remove Terms --ensure yes + +dump + +table_remove Users + +dump From fc2173cf36b64cabf1f2f7d353eee72d803ce7d0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 18:40:06 +0900 Subject: [PATCH 05/29] grn_hook_entry_to_string: add --- include/groonga/groonga.h | 3 +++ lib/db.c | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/include/groonga/groonga.h b/include/groonga/groonga.h index 16fb75881f..787524dde2 100644 --- a/include/groonga/groonga.h +++ b/include/groonga/groonga.h @@ -1048,6 +1048,9 @@ typedef enum { #define GRN_N_HOOK_ENTRIES 5 /* (GRN_HOOK_SELECT + 1) */ +GRN_API const char * +grn_hook_entry_to_string(grn_hook_entry entry); + GRN_API grn_rc grn_obj_add_hook(grn_ctx *ctx, grn_obj *obj, diff --git a/lib/db.c b/lib/db.c index 529f157bd6..41ae986f80 100644 --- a/lib/db.c +++ b/lib/db.c @@ -145,6 +145,25 @@ grn_set_reference_count_enable(bool enable) return GRN_SUCCESS; } +const char * +grn_hook_entry_to_string(grn_hook_entry entry) +{ + switch (entry) { + case GRN_HOOK_SET: + return "set"; + case GRN_HOOK_GET: + return "get"; + case GRN_HOOK_INSERT: + return "insert"; + case GRN_HOOK_DELETE: + return "delete"; + case GRN_HOOK_SELECT: + return "select"; + default: + return "unknown"; + } +} + grn_inline static void gen_pathname(const char *path, char *buffer, int fno) { From 51e6ff1990a253630b6ea3abab86c644d09ca74f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 18:44:23 +0900 Subject: [PATCH 06/29] grn_obj_delete_hook: add DDL:delete_hook log --- lib/db.c | 44 +++++++++++++++++++ .../suite/column_remove/index.expected | 1 + .../db/auto_recovery/column/index.expected | 1 + .../column/indexed_table.expected | 2 + .../column/token_vector.expected | 1 + .../table/column/index_column.expected | 1 + .../suite/wal_recover/ii/online.expected | 1 + 7 files changed, 51 insertions(+) diff --git a/lib/db.c b/lib/db.c index 41ae986f80..dc9395aa4f 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9570,6 +9570,10 @@ grn_obj_delete_hook(grn_ctx *ctx, int offset) { GRN_API_ENTER; + grn_id removed_hook_target_id = GRN_ID_NIL; + char removed_hook_target_name[GRN_TABLE_MAX_KEY_SIZE]; + uint32_t removed_hook_target_name_size = 0; + bool removed = false; { int i = 0; grn_hook *h, **last = &DB_OBJ(obj)->hooks[entry]; @@ -9578,6 +9582,19 @@ grn_obj_delete_hook(grn_ctx *ctx, return GRN_INVALID_ARGUMENT; } if (++i > offset) { + removed = true; + if (h->proc == NULL && + h->hld_size == sizeof(grn_obj_default_set_value_hook_data)) { + grn_obj_default_set_value_hook_data *data = + (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(h); + removed_hook_target_id = data->target; + removed_hook_target_name_size = + grn_table_get_key(ctx, + ctx->impl->db, + data->target, + removed_hook_target_name, + GRN_TABLE_MAX_KEY_SIZE); + } break; } last = &h->next; @@ -9585,6 +9602,33 @@ grn_obj_delete_hook(grn_ctx *ctx, *last = h->next; GRN_FREE(h); } + if (removed) { + grn_id id = DB_OBJ(obj)->id; + uint32_t name_size = 0; + const char *name = _grn_table_key(ctx, ctx->impl->db, id, &name_size); + if (name_size > 0) { + if (removed_hook_target_id == GRN_ID_NIL) { + GRN_LOG(ctx, + GRN_LOG_NOTICE, + "DDL:%u:delete_hook:%s %.*s", + id, + grn_hook_entry_to_string(entry), + (int)name_size, + name); + } else { + GRN_LOG(ctx, + GRN_LOG_NOTICE, + "DDL:%u:delete_hook:%s %.*s %.*s(%u)", + id, + grn_hook_entry_to_string(entry), + (int)name_size, + name, + (int)removed_hook_target_name_size, + removed_hook_target_name, + removed_hook_target_id); + } + } + } grn_obj_spec_save(ctx, DB_OBJ(obj)); GRN_API_RETURN(GRN_SUCCESS); } diff --git a/test/command/suite/column_remove/index.expected b/test/command/suite/column_remove/index.expected index e3b41e42bd..8115272206 100644 --- a/test/command/suite/column_remove/index.expected +++ b/test/command/suite/column_remove/index.expected @@ -16,6 +16,7 @@ column_create Terms users_name COLUMN_INDEX|WITH_POSITION Users name column_remove Terms users_name [[0,0.0,0.0],true] #|n| DDL:259:obj_remove Terms.users_name +#|n| DDL:257:delete_hook:set Users.name Terms.users_name(259) #|n| spec:257:update:Users.name:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.users_name:72(column:index) dump diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected index 79590588e9..52e805023b 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected @@ -41,6 +41,7 @@ _database_reopen #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:259:obj_remove Terms.#broken#data_content +#|n| DDL:258:delete_hook:set Data.content Terms.#broken#data_content(259) #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.#broken#data_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected index 57c1ece0b2..d3205fea7b 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected @@ -42,7 +42,9 @@ _database_reopen #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 #|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:258:obj_remove Terms.#broken#data_key +#|n| DDL:257:delete_hook:insert Data Terms.#broken#data_key(258) #|n| spec:257:update:Data:49(table:pat_key):0 +#|n| DDL:257:delete_hook:delete Data Terms.#broken#data_key(258) #|n| spec:257:update:Data:49(table:pat_key):0 #|n| spec:258:remove:Terms.#broken#data_key:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected index c145f57bbb..abefe04ea4 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected @@ -53,6 +53,7 @@ _database_reopen #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:259:obj_remove Data.#broken#content_terms +#|n| DDL:258:delete_hook:set Data.content Data.#broken#content_terms(259) #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Data.#broken#content_terms:65(column:var_size) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected index 905b937de7..ac17d8f56f 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected @@ -47,6 +47,7 @@ _database_reopen #|n| spec:261:update:#recovering#Data.content:65(column:var_size):14(ShortText) #|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:259:obj_remove Terms.#broken#data_content +#|n| DDL:257:delete_hook:set Data.content Terms.#broken#data_content(259) #|n| spec:257:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.#broken#data_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (262) diff --git a/test/command/suite/wal_recover/ii/online.expected b/test/command/suite/wal_recover/ii/online.expected index 09b700c842..805197d0da 100644 --- a/test/command/suite/wal_recover/ii/online.expected +++ b/test/command/suite/wal_recover/ii/online.expected @@ -85,6 +85,7 @@ _database_reopen #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) #|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:259:obj_remove Terms.#broken#memos_content +#|n| DDL:257:delete_hook:set Memos.content Terms.#broken#memos_content(259) #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) #|n| spec:259:remove:Terms.#broken#memos_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) From 4e4c48d59a5312543fbeb8137218314151385e2c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 1 Jul 2024 18:48:42 +0900 Subject: [PATCH 07/29] remove_index: show unsupported index target's ID --- lib/db.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/db.c b/lib/db.c index dc9395aa4f..75ab8b4310 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9678,9 +9678,10 @@ remove_index(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, uint32_t flags) GRN_TEXT_INIT(&inspected, 0); grn_inspect_limited(ctx, &inspected, target); ERR(GRN_UNKNOWN_ERROR, - "%s[%s] hook has an unsupported index target: %.*s", + "%s[%s] hook has an unsupported index target: %u: %.*s", tag, name, + data->target, (int)GRN_TEXT_LEN(&inspected), GRN_TEXT_VALUE(&inspected)); GRN_OBJ_FIN(ctx, &inspected); From 7185930cee87acb2e3db76e100650d0cdd4f9a23 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 11:01:50 +0900 Subject: [PATCH 08/29] remove_index: show hook entry name --- lib/db.c | 6 ++++-- .../suite/object_remove/broken/indexed_column.expected | 4 ++-- .../object_remove/broken/indexed_column_force.expected | 4 ++-- .../broken/index_column/default.expected | 4 ++-- .../broken/index_column/dependent.expected | 4 ++-- .../broken/index_column/dependent_force.expected | 2 +- .../logical_table_remove/broken/index_column/force.expected | 2 +- .../logical_table_remove/broken/lexicon/default.expected | 4 ++-- .../logical_table_remove/broken/lexicon/dependent.expected | 4 ++-- .../broken/lexicon/dependent_force.expected | 2 +- .../logical_table_remove/broken/lexicon/force.expected | 2 +- .../broken/refer_broken_index_column/column.expected | 2 +- .../broken/refer_broken_index_column/table.expected | 2 +- 13 files changed, 22 insertions(+), 20 deletions(-) diff --git a/lib/db.c b/lib/db.c index 75ab8b4310..15571eca25 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9657,9 +9657,10 @@ remove_index(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, uint32_t flags) hook_name, GRN_TABLE_MAX_KEY_SIZE); ERR(GRN_OBJECT_CORRUPT, - "%s[%s] hook has a dangling reference: <%.*s>(%u)", + "%s[%s][%s] hook has a dangling reference: <%.*s>(%u)", tag, name, + grn_hook_entry_to_string(entry), hook_name_length, hook_name, data->target); @@ -9678,9 +9679,10 @@ remove_index(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, uint32_t flags) GRN_TEXT_INIT(&inspected, 0); grn_inspect_limited(ctx, &inspected, target); ERR(GRN_UNKNOWN_ERROR, - "%s[%s] hook has an unsupported index target: %u: %.*s", + "%s[%s][%s] hook has an unsupported index target: %u: %.*s", tag, name, + grn_hook_entry_to_string(entry), data->target, (int)GRN_TEXT_LEN(&inspected), GRN_TEXT_VALUE(&inspected)); diff --git a/test/command/suite/object_remove/broken/indexed_column.expected b/test/command/suite/object_remove/broken/indexed_column.expected index f63c99ace3..66c5f513a0 100644 --- a/test/command/suite/object_remove/broken/indexed_column.expected +++ b/test/command/suite/object_remove/broken/indexed_column.expected @@ -16,12 +16,12 @@ object_remove Users.name 0.0, 0.0 ], - "[object][remove][index][Users.name] hook has a dangling reference: (259)" + "[object][remove][index][Users.name][set] hook has a dangling reference: (259)" ], false ] #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() -#|e| [object][remove][index][Users.name] hook has a dangling reference: (259) +#|e| [object][remove][index][Users.name][set] hook has a dangling reference: (259) object_exist Users.name [[0,0.0,0.0],true] diff --git a/test/command/suite/object_remove/broken/indexed_column_force.expected b/test/command/suite/object_remove/broken/indexed_column_force.expected index 26abedb7f4..e9cb827bfb 100644 --- a/test/command/suite/object_remove/broken/indexed_column_force.expected +++ b/test/command/suite/object_remove/broken/indexed_column_force.expected @@ -12,10 +12,10 @@ object_remove Users.name --force yes [[0,0.0,0.0],true] #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() -#|e| [object][remove][index][Users.name] hook has a dangling reference: (259) +#|e| [object][remove][index][Users.name][set] hook has a dangling reference: (259) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() -#|e| [object][remove][index][Users.name] hook has a dangling reference: (259) +#|e| [object][remove][index][Users.name][set] hook has a dangling reference: (259) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() column_create Users name COLUMN_SCALAR ShortText diff --git a/test/command/suite/sharding/logical_table_remove/broken/index_column/default.expected b/test/command/suite/sharding/logical_table_remove/broken/index_column/default.expected index f6362e6289..f1c7e35f09 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/index_column/default.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/index_column/default.expected @@ -55,12 +55,12 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border 0.0, 0.0 ], - "object corrupt: <[object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (required: >= 64): #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) object_exist Times_20150204.timestamp_index [[0,0.0,0.0],true] dump diff --git a/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent.expected b/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent.expected index f4391b9340..e25779ac84 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent.expected @@ -55,14 +55,14 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border 0.0, 0.0 ], - "object corrupt: <[object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (required: >= 64): #|e| grn_ctx_at: failed to open object: <276>():<72>() #|e| grn_ctx_at: failed to open object: <276>():<72>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) object_exist Times_20150204.timestamp_index [[0,0.0,0.0],true] dump diff --git a/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent_force.expected b/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent_force.expected index fe5432fc4e..b85bcc8583 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent_force.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/index_column/dependent_force.expected @@ -53,7 +53,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border #|e| grn_ctx_at: failed to open object: <276>():<72>() #|e| grn_ctx_at: failed to open object: <276>():<72>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <276>():<72>() object_exist Times_20150204.timestamp_index diff --git a/test/command/suite/sharding/logical_table_remove/broken/index_column/force.expected b/test/command/suite/sharding/logical_table_remove/broken/index_column/force.expected index fae1d5c6d7..f90a9686ca 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/index_column/force.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/index_column/force.expected @@ -51,7 +51,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border [[0,0.0,0.0],true] #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <276>():<72>() object_exist Times_20150204.timestamp_index diff --git a/test/command/suite/sharding/logical_table_remove/broken/lexicon/default.expected b/test/command/suite/sharding/logical_table_remove/broken/lexicon/default.expected index c4864c1fe3..bfe294f524 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/lexicon/default.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/lexicon/default.expected @@ -55,14 +55,14 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border 0.0, 0.0 ], - "object corrupt: <[object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (required: >= 64): #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) dump plugin_register sharding diff --git a/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent.expected b/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent.expected index 99c138b0ca..29a96068f7 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent.expected @@ -55,7 +55,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border 0.0, 0.0 ], - "object corrupt: <[object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (required: >= 64): @@ -73,7 +73,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) dump plugin_register sharding diff --git a/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent_force.expected b/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent_force.expected index 2b70a7165a..a117389f62 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent_force.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/lexicon/dependent_force.expected @@ -64,7 +64,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() diff --git a/test/command/suite/sharding/logical_table_remove/broken/lexicon/force.expected b/test/command/suite/sharding/logical_table_remove/broken/lexicon/force.expected index 4aa711bd7b..c957a0b472 100644 --- a/test/command/suite/sharding/logical_table_remove/broken/lexicon/force.expected +++ b/test/command/suite/sharding/logical_table_remove/broken/lexicon/force.expected @@ -53,7 +53,7 @@ logical_table_remove Logs timestamp --min "2015-02-04 00:00:00" --min_border #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() #|e| grn_ctx_at: failed to open object: <276>():<72>() -#|e| [object][remove][index][Logs_20150204.timestamp] hook has a dangling reference: (276) +#|e| [object][remove][index][Logs_20150204.timestamp][set] hook has a dangling reference: (276) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| [pat][open] failed to open grn_io: #|e| grn_ctx_at: failed to open object: <275>():<49>() diff --git a/test/command/suite/table_remove/broken/refer_broken_index_column/column.expected b/test/command/suite/table_remove/broken/refer_broken_index_column/column.expected index 437572ca03..e4a693b230 100644 --- a/test/command/suite/table_remove/broken/refer_broken_index_column/column.expected +++ b/test/command/suite/table_remove/broken/refer_broken_index_column/column.expected @@ -12,7 +12,7 @@ table_remove Users --ensure yes [[0,0.0,0.0],true] #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() -#|e| [object][remove][index][Users.name] hook has a dangling reference: (259) +#|e| [object][remove][index][Users.name][set] hook has a dangling reference: (259) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <259>():<72>() dump diff --git a/test/command/suite/table_remove/broken/refer_broken_index_column/table.expected b/test/command/suite/table_remove/broken/refer_broken_index_column/table.expected index 4f96b82ca6..8f038ef713 100644 --- a/test/command/suite/table_remove/broken/refer_broken_index_column/table.expected +++ b/test/command/suite/table_remove/broken/refer_broken_index_column/table.expected @@ -10,7 +10,7 @@ table_remove Users --ensure yes [[0,0.0,0.0],true] #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <258>():<72>() -#|e| [object][remove][index][Users] hook has a dangling reference: (258) +#|e| [object][remove][index][Users][insert] hook has a dangling reference: (258) #|e| [io][open] file size is too small: <8>(required: >= 64): #|e| grn_ctx_at: failed to open object: <258>():<72>() dump From 19575509ddd4813ddd2b310eaf2d94226cebc351 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 11:03:17 +0900 Subject: [PATCH 09/29] Fix style --- lib/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/db.c b/lib/db.c index 15571eca25..9c7854b523 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9583,7 +9583,7 @@ grn_obj_delete_hook(grn_ctx *ctx, } if (++i > offset) { removed = true; - if (h->proc == NULL && + if (!h->proc && h->hld_size == sizeof(grn_obj_default_set_value_hook_data)) { grn_obj_default_set_value_hook_data *data = (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(h); From fe600b3ee3a53a9d74b409204ca095bade49a9d6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 11:33:48 +0900 Subject: [PATCH 10/29] grn_obj_delete_hook: show remained hooks --- lib/db.c | 62 ++++++++++--------- .../suite/column_remove/index.expected | 2 +- .../db/auto_recovery/column/index.expected | 2 +- .../column/indexed_table.expected | 4 +- .../column/token_vector.expected | 2 +- .../table/column/index_column.expected | 2 +- .../suite/wal_recover/ii/online.expected | 2 +- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/lib/db.c b/lib/db.c index 9c7854b523..4bb47a3942 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9570,9 +9570,8 @@ grn_obj_delete_hook(grn_ctx *ctx, int offset) { GRN_API_ENTER; - grn_id removed_hook_target_id = GRN_ID_NIL; - char removed_hook_target_name[GRN_TABLE_MAX_KEY_SIZE]; - uint32_t removed_hook_target_name_size = 0; + grn_obj extra_info; + GRN_TEXT_INIT(&extra_info, 0); bool removed = false; { int i = 0; @@ -9587,13 +9586,9 @@ grn_obj_delete_hook(grn_ctx *ctx, h->hld_size == sizeof(grn_obj_default_set_value_hook_data)) { grn_obj_default_set_value_hook_data *data = (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(h); - removed_hook_target_id = data->target; - removed_hook_target_name_size = - grn_table_get_key(ctx, - ctx->impl->db, - data->target, - removed_hook_target_name, - GRN_TABLE_MAX_KEY_SIZE); + GRN_TEXT_PUTC(ctx, &extra_info, ' '); + grn_table_get_key2(ctx, ctx->impl->db, data->target, &extra_info); + grn_text_printf(ctx, &extra_info, "(%u)", data->target); } break; } @@ -9606,29 +9601,38 @@ grn_obj_delete_hook(grn_ctx *ctx, grn_id id = DB_OBJ(obj)->id; uint32_t name_size = 0; const char *name = _grn_table_key(ctx, ctx->impl->db, id, &name_size); - if (name_size > 0) { - if (removed_hook_target_id == GRN_ID_NIL) { - GRN_LOG(ctx, - GRN_LOG_NOTICE, - "DDL:%u:delete_hook:%s %.*s", - id, - grn_hook_entry_to_string(entry), - (int)name_size, - name); + GRN_TEXT_PUTS(ctx, &extra_info, " ["); + grn_hook *first_hook = DB_OBJ(obj)->hooks[entry]; + grn_hook *hook; + for (hook = first_hook; hook; hook = hook->next) { + if (hook != first_hook) { + GRN_TEXT_PUTC(ctx, &extra_info, ','); + } + if (!hook->proc && + hook->hld_size == sizeof(grn_obj_default_set_value_hook_data)) { + grn_obj_default_set_value_hook_data *data = + (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(hook); + grn_table_get_key2(ctx, ctx->impl->db, data->target, &extra_info); + grn_text_printf(ctx, &extra_info, "(%u)", data->target); } else { - GRN_LOG(ctx, - GRN_LOG_NOTICE, - "DDL:%u:delete_hook:%s %.*s %.*s(%u)", - id, - grn_hook_entry_to_string(entry), - (int)name_size, - name, - (int)removed_hook_target_name_size, - removed_hook_target_name, - removed_hook_target_id); + grn_id hook_proc_id = DB_OBJ(hook->proc)->id; + grn_table_get_key2(ctx, ctx->impl->db, hook_proc_id, &extra_info); + grn_text_printf(ctx, &extra_info, "(%u)", hook_proc_id); } } + GRN_TEXT_PUTS(ctx, &extra_info, "]"); + GRN_LOG(ctx, + GRN_LOG_NOTICE, + "DDL:%u:delete_hook:%s%s%.*s%.*s", + id, + grn_hook_entry_to_string(entry), + name_size == 0 ? "" : " ", + (int)name_size, + name, + (int)GRN_TEXT_LEN(&extra_info), + GRN_TEXT_VALUE(&extra_info)); } + GRN_OBJ_FIN(ctx, &extra_info); grn_obj_spec_save(ctx, DB_OBJ(obj)); GRN_API_RETURN(GRN_SUCCESS); } diff --git a/test/command/suite/column_remove/index.expected b/test/command/suite/column_remove/index.expected index 8115272206..ac65478789 100644 --- a/test/command/suite/column_remove/index.expected +++ b/test/command/suite/column_remove/index.expected @@ -16,7 +16,7 @@ column_create Terms users_name COLUMN_INDEX|WITH_POSITION Users name column_remove Terms users_name [[0,0.0,0.0],true] #|n| DDL:259:obj_remove Terms.users_name -#|n| DDL:257:delete_hook:set Users.name Terms.users_name(259) +#|n| DDL:257:delete_hook:set Users.name Terms.users_name(259) [] #|n| spec:257:update:Users.name:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.users_name:72(column:index) dump diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected index 52e805023b..e512571bf2 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected @@ -41,7 +41,7 @@ _database_reopen #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:259:obj_remove Terms.#broken#data_content -#|n| DDL:258:delete_hook:set Data.content Terms.#broken#data_content(259) +#|n| DDL:258:delete_hook:set Data.content Terms.#broken#data_content(259) [Terms.data_content(260)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.#broken#data_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected index d3205fea7b..012d139dc1 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected @@ -42,9 +42,9 @@ _database_reopen #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 #|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:258:obj_remove Terms.#broken#data_key -#|n| DDL:257:delete_hook:insert Data Terms.#broken#data_key(258) +#|n| DDL:257:delete_hook:insert Data Terms.#broken#data_key(258) [] #|n| spec:257:update:Data:49(table:pat_key):0 -#|n| DDL:257:delete_hook:delete Data Terms.#broken#data_key(258) +#|n| DDL:257:delete_hook:delete Data Terms.#broken#data_key(258) [] #|n| spec:257:update:Data:49(table:pat_key):0 #|n| spec:258:remove:Terms.#broken#data_key:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected index abefe04ea4..1c41436d34 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected @@ -53,7 +53,7 @@ _database_reopen #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:259:obj_remove Data.#broken#content_terms -#|n| DDL:258:delete_hook:set Data.content Data.#broken#content_terms(259) +#|n| DDL:258:delete_hook:set Data.content Data.#broken#content_terms(259) [Data.content_terms(260)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Data.#broken#content_terms:65(column:var_size) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) diff --git a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected index ac17d8f56f..1b3f2f727f 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected @@ -47,7 +47,7 @@ _database_reopen #|n| spec:261:update:#recovering#Data.content:65(column:var_size):14(ShortText) #|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:259:obj_remove Terms.#broken#data_content -#|n| DDL:257:delete_hook:set Data.content Terms.#broken#data_content(259) +#|n| DDL:257:delete_hook:set Data.content Terms.#broken#data_content(259) [] #|n| spec:257:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:259:remove:Terms.#broken#data_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (262) diff --git a/test/command/suite/wal_recover/ii/online.expected b/test/command/suite/wal_recover/ii/online.expected index 805197d0da..5407d7dab1 100644 --- a/test/command/suite/wal_recover/ii/online.expected +++ b/test/command/suite/wal_recover/ii/online.expected @@ -85,7 +85,7 @@ _database_reopen #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) #|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:259:obj_remove Terms.#broken#memos_content -#|n| DDL:257:delete_hook:set Memos.content Terms.#broken#memos_content(259) +#|n| DDL:257:delete_hook:set Memos.content Terms.#broken#memos_content(259) [Terms.memos_content(260)] #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) #|n| spec:259:remove:Terms.#broken#memos_content:72(column:index) #|n| [db][wal][recover] succeeded to rebuild broken column: (260) From 1077c7d03b1cdc242a3cff95181342b5746a3dfd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 11:40:08 +0900 Subject: [PATCH 11/29] test: increase timeout for stable CI --- test/command/suite/table_remove/broken/index_column.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/command/suite/table_remove/broken/index_column.test b/test/command/suite/table_remove/broken/index_column.test index 108c1ba53d..c98d9a40e9 100644 --- a/test/command/suite/table_remove/broken/index_column.test +++ b/test/command/suite/table_remove/broken/index_column.test @@ -18,4 +18,6 @@ dump table_remove Users +#@timeout 60 dump +#@timeout default From 264be86cf8853ab9d6b8d88e5a21390664a030a9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 11:56:52 +0900 Subject: [PATCH 12/29] grn_obj_remove: change remove order Before: 1. Remove paths of the target object 2. Remove metadata of the target object After: 1. Remove metadata of the target object 2. Remove paths of the target object The before order may cause "there is metadata but path doesn't exist". --- lib/db.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/db.c b/lib/db.c index 4bb47a3942..e3a5ef9bd0 100644 --- a/lib/db.c +++ b/lib/db.c @@ -10306,12 +10306,12 @@ grn_obj_remove_pat(grn_ctx *ctx, } if (path) { - rc = grn_pat_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_pat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10363,12 +10363,12 @@ grn_obj_remove_dat(grn_ctx *ctx, } if (path) { - rc = grn_dat_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_dat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10420,12 +10420,12 @@ grn_obj_remove_hash(grn_ctx *ctx, } if (path) { - rc = grn_hash_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_hash_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10473,12 +10473,12 @@ grn_obj_remove_array(grn_ctx *ctx, } if (path) { - rc = grn_array_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_array_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10515,12 +10515,12 @@ grn_obj_remove_ja(grn_ctx *ctx, } if (path) { - rc = grn_ja_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ja_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10556,12 +10556,12 @@ grn_obj_remove_ra(grn_ctx *ctx, } if (path) { - rc = grn_ra_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ra_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } @@ -10594,12 +10594,12 @@ grn_obj_remove_index(grn_ctx *ctx, } if (path) { - rc = grn_ii_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ii_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } From 13abaf40d58118a0671c70bae9cec05a664cbcf0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 13:09:56 +0900 Subject: [PATCH 13/29] del_hook: remove all hooks that refer the target ID delete_source_hook() is only used when the target object is removed. So we can simplify like this. If we want to use this when the target object isn't removed, we may want to change this implementation or create another one. --- lib/db.c | 61 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/lib/db.c b/lib/db.c index e3a5ef9bd0..c40833f4bf 100644 --- a/lib/db.c +++ b/lib/db.c @@ -8054,39 +8054,47 @@ update_source_hook(grn_ctx *ctx, grn_obj *obj) } static void -del_hook(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, grn_obj *hld) +del_hook(grn_ctx *ctx, grn_obj *obj, grn_hook_entry entry, grn_id target_id) { + grn_obj offsets; + GRN_INT32_INIT(&offsets, GRN_OBJ_VECTOR); + int i; - void *hld_value = NULL; - uint32_t hld_size = 0; - grn_hook **last; - hld_value = GRN_BULK_HEAD(hld); - hld_size = GRN_BULK_VSIZE(hld); - if (!hld_size) { - return; - } - for (i = 0, last = &DB_OBJ(obj)->hooks[entry]; *last; - i++, last = &(*last)->next) { - if (!memcmp(GRN_NEXT_ADDR(*last), hld_value, hld_size)) { - grn_obj_delete_hook(ctx, obj, entry, i); - return; + grn_hook *hook; + for (i = 0, hook = DB_OBJ(obj)->hooks[entry]; hook; i++, hook = hook->next) { + if (hook->proc) { + continue; + } + if (hook->hld_size != sizeof(grn_obj_default_set_value_hook_data)) { + continue; + } + grn_obj_default_set_value_hook_data *data = + (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(hook); + if (data->target == target_id) { + GRN_INT32_PUT(ctx, &offsets, i); } } + + int n = GRN_INT32_VECTOR_SIZE(&offsets); + for (i = 0; i < n; i++) { + /* This must be larger -> smaller order. If we use smaller -> + * larger order, offset is changed. */ + int offset = GRN_INT32_VALUE_AT(&offsets, n - i - 1); + grn_obj_delete_hook(ctx, obj, entry, offset); + } + + GRN_OBJ_FIN(ctx, &offsets); } static void delete_source_hook(grn_ctx *ctx, grn_obj *obj) { - grn_id *s = DB_OBJ(obj)->source; + grn_id id = DB_OBJ(obj)->id; + grn_id *source_ids = DB_OBJ(obj)->source; int i, n = DB_OBJ(obj)->source_size / sizeof(grn_id); - grn_obj_default_set_value_hook_data hook_data = {DB_OBJ(obj)->id, 0}; - grn_obj *source, data; - GRN_TEXT_INIT(&data, GRN_OBJ_DO_SHALLOW_COPY); - GRN_TEXT_SET_REF(&data, &hook_data, sizeof(hook_data)); - for (i = 1; i <= n; i++, s++) { - hook_data.section = i; - - source = grn_ctx_at(ctx, *s); + for (i = 0; i < n; i++) { + grn_id source_id = source_ids[i]; + grn_obj *source = grn_ctx_at(ctx, source_id); if (!source) { ERRCLR(ctx); continue; @@ -8096,19 +8104,18 @@ delete_source_hook(grn_ctx *ctx, grn_obj *obj) case GRN_TABLE_HASH_KEY: case GRN_TABLE_PAT_KEY: case GRN_TABLE_DAT_KEY: - del_hook(ctx, source, GRN_HOOK_INSERT, &data); - del_hook(ctx, source, GRN_HOOK_DELETE, &data); + del_hook(ctx, source, GRN_HOOK_INSERT, id); + del_hook(ctx, source, GRN_HOOK_DELETE, id); break; case GRN_COLUMN_FIX_SIZE: case GRN_COLUMN_VAR_SIZE: - del_hook(ctx, source, GRN_HOOK_SET, &data); + del_hook(ctx, source, GRN_HOOK_SET, id); break; default: /* invalid target */ break; } } - grn_obj_close(ctx, &data); } /* This should be used only when the target object can't be opened. If From 5845ad48c1276a3769f9c15fd62f81959926cc6d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 14:23:52 +0900 Subject: [PATCH 14/29] wal recover: ensure removing hooks of broken recovering index --- lib/db_wal_recover.cpp | 11 +-- .../wal_recover/ii/recovering_broken.expected | 68 +++++++++++++++++++ .../wal_recover/ii/recovering_broken.test | 50 ++++++++++++++ 3 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 test/command/suite/wal_recover/ii/recovering_broken.expected create mode 100644 test/command/suite/wal_recover/ii/recovering_broken.test diff --git a/lib/db_wal_recover.cpp b/lib/db_wal_recover.cpp index 86d32836c2..a28fded5dd 100644 --- a/lib/db_wal_recover.cpp +++ b/lib/db_wal_recover.cpp @@ -39,14 +39,9 @@ grn_db_wal_recover_ensure_remove_by_id(grn_ctx *ctx, grn_db *db, grn_id id) return; } - grn_obj_delete_by_id(ctx, (grn_obj *)db, id, true); - - char path[PATH_MAX]; - grn_obj_path_by_id(ctx, (grn_obj *)db, id, path); - grn_io_remove_if_exist(ctx, path); - grn_wal_remove(ctx, path, grn_db_wal_recover_tag); - grn_strcat(path, PATH_MAX, ".c"); - grn_io_remove_if_exist(ctx, path); + grn_ctx_remove_by_id(ctx, + id, + GRN_OBJ_REMOVE_DEPENDENT | GRN_OBJ_REMOVE_ENSURE); } static void diff --git a/test/command/suite/wal_recover/ii/recovering_broken.expected b/test/command/suite/wal_recover/ii/recovering_broken.expected new file mode 100644 index 0000000000..13906e3e30 --- /dev/null +++ b/test/command/suite/wal_recover/ii/recovering_broken.expected @@ -0,0 +1,68 @@ +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR Text +[[0,0.0,0.0],true] +table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenNgram --normalizer NormalizerNFKC130 +[[0,0.0,0.0],true] +column_create Terms memos_content COLUMN_INDEX|WITH_POSITION Memos content +[[0,0.0,0.0],true] +column_create Terms #recovering#memos_content COLUMN_INDEX|WITH_POSITION Memos content +[[0,0.0,0.0],true] +load --table Memos +[ +{"content": "hello world"}, +{"content": "good-by world"} +] +[[0,0.0,0.0],2] +select Memos --query content:@hello +[[0,0.0,0.0],[[[1],[["_id","UInt32"],["content","Text"]],[1,"hello world"]]]] +io_flush +[[0,0.0,0.0],true] +_database_close +[[0,0.0,0.0],true] +_database_reopen +[[0,0.0,0.0],true] +#|e| [column][index] file type must be 0x48: <0000> +#|e| grn_ctx_at: failed to open object: <260>():<72>() +#|e| grn_ctx_at: failed to open object: <260>():<72>() +#|n| DDL:257:delete_hook:set Memos.content Terms.#recovering#memos_content(260) [Terms.memos_content(259)] +#|n| spec:257:update:Memos.content:65(column:var_size):15(Text) +select Memos --query content:@hello +[[0,0.0,0.0],[[[1],[["_id","UInt32"],["content","Text"]],[1,"hello world"]]]] +dump +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR Text + +table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenNgram --normalizer NormalizerNFKC130 + +load --table Memos +[ +["_id","content"], +[1,"hello world"], +[2,"good-by world"] +] + +column_create Terms memos_content COLUMN_INDEX|WITH_POSITION Memos content +column_remove Memos content +[[0,0.0,0.0],true] +dump +table_create Memos TABLE_NO_KEY + +table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenNgram --normalizer NormalizerNFKC130 + +load --table Memos +[ +["_id"], +[1], +[2] +] + +load --table Terms +[ +["_key"], +["-"], +["by"], +["good"], +["hello"], +["world"] +] diff --git a/test/command/suite/wal_recover/ii/recovering_broken.test b/test/command/suite/wal_recover/ii/recovering_broken.test new file mode 100644 index 0000000000..7f9342598a --- /dev/null +++ b/test/command/suite/wal_recover/ii/recovering_broken.test @@ -0,0 +1,50 @@ +# _database_close/_database_reopen can't use with HTTP because requests via HTTP are always +# processed by workers. Workers never use primary WAL role. +#@require-interface stdio + +#@require-feature message_pack + +#$GROONGA_ALLOW_DATABASE_REOPEN=yes +#$GRN_ENABLE_REFERENCE_COUNT=no +#$GRN_WAL_ROLE=primary + +#@disable-logging +cache_limit --max 0 +#@enable-logging + +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR Text + +table_create Terms TABLE_PAT_KEY ShortText \ + --default_tokenizer TokenNgram \ + --normalizer NormalizerNFKC130 +column_create Terms memos_content COLUMN_INDEX|WITH_POSITION Memos content +column_create Terms #recovering#memos_content \ + COLUMN_INDEX|WITH_POSITION Memos content + +load --table Memos +[ +{"content": "hello world"}, +{"content": "good-by world"} +] + +select Memos --query content:@hello + +io_flush + +_database_close + +# Break Terms.#recovering#memos_content +#@copy-path #{db_path}.0000104.c #{db_path}.0000104 + +#@add-important-log-levels notice +_database_reopen +#@remove-important-log-levels notice + +select Memos --query content:@hello + +dump + +column_remove Memos content + +dump From 005cab436132de57aae12f54a271a2f7e5d87af6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 14:55:52 +0900 Subject: [PATCH 15/29] grn_obj_add_hook: add DDL: log --- lib/db.c | 37 +++++++++++++++++++ .../column_create/index/source/log.expected | 2 + .../db/auto_recovery/column/index.expected | 1 + .../column/indexed_column.expected | 1 + .../column/indexed_table.expected | 2 + .../column/token_vector.expected | 1 + .../table/column/index_column.expected | 1 + .../suite/wal_recover/ii/online.expected | 1 + 8 files changed, 46 insertions(+) diff --git a/lib/db.c b/lib/db.c index c40833f4bf..68a725caf1 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9528,6 +9528,43 @@ grn_obj_add_hook(grn_ctx *ctx, } new->next = *last; *last = new; + if (grn_logger_pass(ctx, GRN_LOG_NOTICE)) { + grn_id id = DB_OBJ(obj)->id; + uint32_t name_size = 0; + const char *name = _grn_table_key(ctx, ctx->impl->db, id, &name_size); + grn_obj extra_info; + GRN_TEXT_INIT(&extra_info, 0); + GRN_TEXT_PUTS(ctx, &extra_info, " ["); + grn_hook *hook; + for (hook = new; hook; hook = hook->next) { + if (hook != new) { + GRN_TEXT_PUTC(ctx, &extra_info, ','); + } + if (!hook->proc && + hook->hld_size == sizeof(grn_obj_default_set_value_hook_data)) { + grn_obj_default_set_value_hook_data *data = + (grn_obj_default_set_value_hook_data *)GRN_NEXT_ADDR(hook); + grn_table_get_key2(ctx, ctx->impl->db, data->target, &extra_info); + grn_text_printf(ctx, &extra_info, "(%u)", data->target); + } else { + grn_id hook_proc_id = DB_OBJ(hook->proc)->id; + grn_table_get_key2(ctx, ctx->impl->db, hook_proc_id, &extra_info); + grn_text_printf(ctx, &extra_info, "(%u)", hook_proc_id); + } + } + GRN_TEXT_PUTS(ctx, &extra_info, "]"); + GRN_LOG(ctx, + GRN_LOG_NOTICE, + "DDL:%u:add_hook:%s%s%.*s%.*s", + id, + grn_hook_entry_to_string(entry), + name_size == 0 ? "" : " ", + (int)name_size, + name, + (int)GRN_TEXT_LEN(&extra_info), + GRN_TEXT_VALUE(&extra_info)); + GRN_OBJ_FIN(ctx, &extra_info); + } grn_obj_spec_save(ctx, DB_OBJ(obj)); } exit: diff --git a/test/command/suite/column_create/index/source/log.expected b/test/command/suite/column_create/index/source/log.expected index 799e5018ad..e6d1fe6e2d 100644 --- a/test/command/suite/column_create/index/source/log.expected +++ b/test/command/suite/column_create/index/source/log.expected @@ -11,6 +11,8 @@ column_create Lexicon data_values COLUMN_INDEX|WITH_SECTION|WITH_POSITION Data #|n| DDL:260:column_create Lexicon data_values #|n| spec:260:update:Lexicon.data_values:72(column:index):256(Data) #|n| DDL:260:set_source Lexicon.data_values Data.value1,Data.value2 +#|n| DDL:257:add_hook:set Data.value1 [Lexicon.data_values(260)] #|n| spec:257:update:Data.value1:65(column:var_size):15(Text) +#|n| DDL:258:add_hook:set Data.value2 [Lexicon.data_values(260)] #|n| spec:258:update:Data.value2:65(column:var_size):15(Text) #|n| spec:260:update:Lexicon.data_values:72(column:index):256(Data) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected index e512571bf2..c59d99e955 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected @@ -38,6 +38,7 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#data_content #|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:260:set_source Terms.#recovering#data_content Data.content +#|n| DDL:258:add_hook:set Data.content [Terms.#recovering#data_content(260),Terms.data_content(259)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:259:obj_remove Terms.#broken#data_content diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected index 9012c40b10..9690a4f1f1 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected @@ -41,6 +41,7 @@ _database_reopen #|n| DDL:261:column_create Terms #recovering#data_content #|n| spec:261:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:261:set_source Terms.#recovering#data_content Data.#recovering#content +#|n| DDL:260:add_hook:set Data.#recovering#content [Terms.#recovering#data_content(261)] #|n| spec:260:update:Data.#recovering#content:65(column:var_size):14(ShortText) #|n| spec:261:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:258:obj_remove Data.#broken#content diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected index 012d139dc1..a8f2904947 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected @@ -38,7 +38,9 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#data_key #|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:260:set_source Terms.#recovering#data_key #recovering#Data +#|n| DDL:259:add_hook:insert #recovering#Data [Terms.#recovering#data_key(260)] #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 +#|n| DDL:259:add_hook:delete #recovering#Data [Terms.#recovering#data_key(260)] #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 #|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:258:obj_remove Terms.#broken#data_key diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected index 1c41436d34..75ee15e141 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected @@ -50,6 +50,7 @@ _database_reopen #|n| DDL:260:column_create Data #recovering#content_terms #|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:260:set_source Data.#recovering#content_terms Data.content +#|n| DDL:258:add_hook:set Data.content [Data.#recovering#content_terms(260),Data.content_terms(259)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) #|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:259:obj_remove Data.#broken#content_terms diff --git a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected index 1b3f2f727f..c5929af592 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected @@ -44,6 +44,7 @@ _database_reopen #|n| DDL:262:column_create Terms #recovering#data_content #|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:262:set_source Terms.#recovering#data_content #recovering#Data.content +#|n| DDL:261:add_hook:set #recovering#Data.content [Terms.#recovering#data_content(262)] #|n| spec:261:update:#recovering#Data.content:65(column:var_size):14(ShortText) #|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:259:obj_remove Terms.#broken#data_content diff --git a/test/command/suite/wal_recover/ii/online.expected b/test/command/suite/wal_recover/ii/online.expected index 5407d7dab1..47c8176364 100644 --- a/test/command/suite/wal_recover/ii/online.expected +++ b/test/command/suite/wal_recover/ii/online.expected @@ -82,6 +82,7 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#memos_content #|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:260:set_source Terms.#recovering#memos_content Memos.content +#|n| DDL:257:add_hook:set Memos.content [Terms.#recovering#memos_content(260),Terms.memos_content(259)] #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) #|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:259:obj_remove Terms.#broken#memos_content From 201a437414af94e8153fb5d7c1a9fa5b8f0f2eef Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 17:50:09 +0900 Subject: [PATCH 16/29] wal recover: ensure removing dependent objects of broken objects --- lib/db_wal_recover.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/db_wal_recover.cpp b/lib/db_wal_recover.cpp index a28fded5dd..5498f951bd 100644 --- a/lib/db_wal_recover.cpp +++ b/lib/db_wal_recover.cpp @@ -55,7 +55,9 @@ grn_db_wal_recover_remove_object(grn_ctx *ctx, id = DB_OBJ(object)->id; } grn_obj_clear_lock(ctx, object); - grn_obj_remove(ctx, object); + grn_obj_remove_flags(ctx, + object, + GRN_OBJ_REMOVE_DEPENDENT | GRN_OBJ_REMOVE_ENSURE); } grn_db_wal_recover_ensure_remove_by_id(ctx, db, id); } From 710e938e510911d73784659ddacf9b2265a8b400 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 17:52:39 +0900 Subject: [PATCH 17/29] grn_obj_set_info_source: use more crash safe save order Before: 1. Save hooks of sources 2. Save sources of the index column After: 1. Save sources of the index column 2. Save hooks of sources If Groonga is crashed between 1. and 2. with the before order, there are hooks of sources but index column doesn't have sources. In WAL recovery process, index column will be regenerated by removing the index column and create a new index column. In the "removing the index column" process, hooks of sources aren't removed. Because the removing process finds hooks of sources by sources of the index column. If hooks of sources are remained, they break something because they are dangling references. If we use the after order, dangling hooks of sources aren't remained. --- lib/db.c | 3 ++- test/command/suite/column_create/index/source/log.expected | 2 +- .../suite/wal_recover/db/auto_recovery/column/index.expected | 2 +- .../db/auto_recovery/column/indexed_column.expected | 2 +- .../wal_recover/db/auto_recovery/column/indexed_table.expected | 2 +- .../wal_recover/db/auto_recovery/column/token_vector.expected | 2 +- .../db/auto_recovery/table/column/index_column.expected | 2 +- test/command/suite/wal_recover/ii/online.expected | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/db.c b/lib/db.c index 68a725caf1..571cf75e5e 100644 --- a/lib/db.c +++ b/lib/db.c @@ -8767,6 +8767,7 @@ grn_obj_set_info_source_update(grn_ctx *ctx, grn_obj *obj, grn_obj *value) } DB_OBJ(obj)->source = v2; DB_OBJ(obj)->source_size = s; + grn_obj_spec_save(ctx, DB_OBJ(obj)); switch (obj->header.type) { case GRN_COLUMN_VAR_SIZE: @@ -8783,6 +8784,7 @@ grn_obj_set_info_source_update(grn_ctx *ctx, grn_obj *obj, grn_obj *value) } else { DB_OBJ(obj)->source = NULL; DB_OBJ(obj)->source_size = 0; + grn_obj_spec_save(ctx, DB_OBJ(obj)); } return GRN_SUCCESS; @@ -8802,7 +8804,6 @@ grn_obj_set_info_source(grn_ctx *ctx, grn_obj *obj, grn_obj *value) if (rc != GRN_SUCCESS) { return rc; } - grn_obj_spec_save(ctx, DB_OBJ(obj)); return rc; } diff --git a/test/command/suite/column_create/index/source/log.expected b/test/command/suite/column_create/index/source/log.expected index e6d1fe6e2d..9015441c55 100644 --- a/test/command/suite/column_create/index/source/log.expected +++ b/test/command/suite/column_create/index/source/log.expected @@ -11,8 +11,8 @@ column_create Lexicon data_values COLUMN_INDEX|WITH_SECTION|WITH_POSITION Data #|n| DDL:260:column_create Lexicon data_values #|n| spec:260:update:Lexicon.data_values:72(column:index):256(Data) #|n| DDL:260:set_source Lexicon.data_values Data.value1,Data.value2 +#|n| spec:260:update:Lexicon.data_values:72(column:index):256(Data) #|n| DDL:257:add_hook:set Data.value1 [Lexicon.data_values(260)] #|n| spec:257:update:Data.value1:65(column:var_size):15(Text) #|n| DDL:258:add_hook:set Data.value2 [Lexicon.data_values(260)] #|n| spec:258:update:Data.value2:65(column:var_size):15(Text) -#|n| spec:260:update:Lexicon.data_values:72(column:index):256(Data) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected index c59d99e955..19936ff3ef 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/index.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/index.expected @@ -38,9 +38,9 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#data_content #|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:260:set_source Terms.#recovering#data_content Data.content +#|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:258:add_hook:set Data.content [Terms.#recovering#data_content(260),Terms.data_content(259)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) -#|n| spec:260:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:259:obj_remove Terms.#broken#data_content #|n| DDL:258:delete_hook:set Data.content Terms.#broken#data_content(259) [Terms.data_content(260)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected index 9690a4f1f1..20cd77428e 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_column.expected @@ -41,9 +41,9 @@ _database_reopen #|n| DDL:261:column_create Terms #recovering#data_content #|n| spec:261:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:261:set_source Terms.#recovering#data_content Data.#recovering#content +#|n| spec:261:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:260:add_hook:set Data.#recovering#content [Terms.#recovering#data_content(261)] #|n| spec:260:update:Data.#recovering#content:65(column:var_size):14(ShortText) -#|n| spec:261:update:Terms.#recovering#data_content:72(column:index):257(Data) #|n| DDL:258:obj_remove Data.#broken#content #|n| DDL:259:obj_remove Terms.data_content #|n| spec:259:remove:Terms.data_content:72(column:index) diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected index a8f2904947..4e796f4f74 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/indexed_table.expected @@ -38,11 +38,11 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#data_key #|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:260:set_source Terms.#recovering#data_key #recovering#Data +#|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:259:add_hook:insert #recovering#Data [Terms.#recovering#data_key(260)] #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 #|n| DDL:259:add_hook:delete #recovering#Data [Terms.#recovering#data_key(260)] #|n| spec:259:update:#recovering#Data:49(table:pat_key):0 -#|n| spec:260:update:Terms.#recovering#data_key:72(column:index):259(#recovering#Data) #|n| DDL:258:obj_remove Terms.#broken#data_key #|n| DDL:257:delete_hook:insert Data Terms.#broken#data_key(258) [] #|n| spec:257:update:Data:49(table:pat_key):0 diff --git a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected index 75ee15e141..0c564b0b28 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/column/token_vector.expected @@ -50,9 +50,9 @@ _database_reopen #|n| DDL:260:column_create Data #recovering#content_terms #|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:260:set_source Data.#recovering#content_terms Data.content +#|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:258:add_hook:set Data.content [Data.#recovering#content_terms(260),Data.content_terms(259)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) -#|n| spec:260:update:Data.#recovering#content_terms:65(column:var_size):256(Terms) #|n| DDL:259:obj_remove Data.#broken#content_terms #|n| DDL:258:delete_hook:set Data.content Data.#broken#content_terms(259) [Data.content_terms(260)] #|n| spec:258:update:Data.content:65(column:var_size):14(ShortText) diff --git a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected index c5929af592..fe282ba617 100644 --- a/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected +++ b/test/command/suite/wal_recover/db/auto_recovery/table/column/index_column.expected @@ -44,9 +44,9 @@ _database_reopen #|n| DDL:262:column_create Terms #recovering#data_content #|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:262:set_source Terms.#recovering#data_content #recovering#Data.content +#|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:261:add_hook:set #recovering#Data.content [Terms.#recovering#data_content(262)] #|n| spec:261:update:#recovering#Data.content:65(column:var_size):14(ShortText) -#|n| spec:262:update:Terms.#recovering#data_content:72(column:index):260(#recovering#Data) #|n| DDL:259:obj_remove Terms.#broken#data_content #|n| DDL:257:delete_hook:set Data.content Terms.#broken#data_content(259) [] #|n| spec:257:update:Data.content:65(column:var_size):14(ShortText) diff --git a/test/command/suite/wal_recover/ii/online.expected b/test/command/suite/wal_recover/ii/online.expected index 47c8176364..4869415749 100644 --- a/test/command/suite/wal_recover/ii/online.expected +++ b/test/command/suite/wal_recover/ii/online.expected @@ -82,9 +82,9 @@ _database_reopen #|n| DDL:260:column_create Terms #recovering#memos_content #|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:260:set_source Terms.#recovering#memos_content Memos.content +#|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:257:add_hook:set Memos.content [Terms.#recovering#memos_content(260),Terms.memos_content(259)] #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) -#|n| spec:260:update:Terms.#recovering#memos_content:72(column:index):256(Memos) #|n| DDL:259:obj_remove Terms.#broken#memos_content #|n| DDL:257:delete_hook:set Memos.content Terms.#broken#memos_content(259) [Terms.memos_content(260)] #|n| spec:257:update:Memos.content:65(column:var_size):15(Text) From 1351e77505d281e6a8caa883ceeada6193c9df80 Mon Sep 17 00:00:00 2001 From: Horimoto Yasuhiro Date: Wed, 3 Jul 2024 09:18:25 +0900 Subject: [PATCH 18/29] packages yum: drop support for CentOS 7 (#1809) GitHub: fix GH-1808 --- .github/workflows/package.yml | 6 ----- packages/Rakefile | 1 - packages/yum/centos-7/Dockerfile | 44 -------------------------------- 3 files changed, 51 deletions(-) delete mode 100644 packages/yum/centos-7/Dockerfile diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 7b74ad4158..e804fdd5b0 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -185,7 +185,6 @@ jobs: - Debian GNU/Linux bookworm arm64 - Debian GNU/Linux trixie amd64 - Debian GNU/Linux trixie arm64 - - CentOS 7 x86_64 - AlmaLinux 8 x86_64 - AlmaLinux 8 aarch64 - AlmaLinux 9 x86_64 @@ -221,11 +220,6 @@ jobs: task-namespace: apt target: debian-trixie-arm64 test-docker-image: arm64v8/debian:trixie - - label: CentOS 7 x86_64 - id: centos-7-x86_64 - task-namespace: yum - target: centos-7 - test-docker-image: centos:7 - label: AlmaLinux 8 x86_64 id: almalinux-8-x86_64 task-namespace: yum diff --git a/packages/Rakefile b/packages/Rakefile index 71a867aceb..2a667c390d 100644 --- a/packages/Rakefile +++ b/packages/Rakefile @@ -83,7 +83,6 @@ class GroongaPackageTask < PackagesGroongaOrgPackageTask "almalinux-8-aarch64", "almalinux-9", "almalinux-9-aarch64", - "centos-7", ] end diff --git a/packages/yum/centos-7/Dockerfile b/packages/yum/centos-7/Dockerfile deleted file mode 100644 index 599674a532..0000000000 --- a/packages/yum/centos-7/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -ARG FROM=centos:7 -FROM ${FROM} - -ENV \ - SCL=devtoolset-11 - -ARG DEBUG -ARG APACHE_ARROW_VERSION - -RUN \ - quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ - yum install -y https://packages.groonga.org/centos/7/groonga-release-latest.noarch.rpm && \ - yum update -y ${quiet} && \ - yum install -y ${quiet} epel-release && \ - yum install -y ${quiet} centos-release-scl && \ - yum groupinstall -y ${quiet} "Development Tools" && \ - yum install -y ${quiet} \ - ${SCL}-gcc-c++ \ - ${SCL}-make \ - arrow-devel-${APACHE_ARROW_VERSION} \ - ccache \ - cmake3 \ - intltool \ - libcurl-devel \ - libedit-devel \ - libevent-devel \ - libstemmer-devel \ - libzstd-devel \ - lz4-devel \ - mecab \ - mecab-devel \ - msgpack-devel \ - openssl-devel \ - pcre-devel \ - php-devel \ - pkgconfig \ - python2-devel \ - ruby \ - tar \ - which \ - xxhash-devel \ - zeromq3-devel \ - zlib-devel && \ - yum clean ${quiet} all From a69fd43260c35deec72e014d52174fc1ed2c49e1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 2 Jul 2024 18:19:19 +0900 Subject: [PATCH 19/29] Revert "grn_obj_remove: change remove order" This reverts commit 264be86cf8853ab9d6b8d88e5a21390664a030a9. If there are path but aren't metadata, we can't remove the remained path. --- lib/db.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/db.c b/lib/db.c index 571cf75e5e..44e186a58d 100644 --- a/lib/db.c +++ b/lib/db.c @@ -10351,12 +10351,12 @@ grn_obj_remove_pat(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_pat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_pat_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10408,12 +10408,12 @@ grn_obj_remove_dat(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_dat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_dat_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10465,12 +10465,12 @@ grn_obj_remove_hash(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_hash_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_hash_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10518,12 +10518,12 @@ grn_obj_remove_array(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_array_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_array_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10560,12 +10560,12 @@ grn_obj_remove_ja(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ja_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_ja_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10601,12 +10601,12 @@ grn_obj_remove_ra(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ra_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_ra_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } @@ -10639,12 +10639,12 @@ grn_obj_remove_index(grn_ctx *ctx, } if (path) { - grn_obj_remove_log_spec_remove(ctx, db, id, type); - rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); + rc = grn_ii_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; } - rc = grn_ii_remove(ctx, path); + grn_obj_remove_log_spec_remove(ctx, db, id, type); + rc = grn_obj_delete_by_id(ctx, db, id, GRN_TRUE); if (rc != GRN_SUCCESS) { return rc; } From f4a8f9b3285bb01c0fc286530f4a9b567a7b4c13 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 3 Jul 2024 09:34:56 +0900 Subject: [PATCH 20/29] wal recover: introduce removing mark to ensure removing an object We needs the following steps on removing an object: 1. Remove paths for the object 2. Remove metadata for the object If Groonga is crashed between 1. and 2., WAL recovery process can't determine whether the object should be recovered or removed. This creates "#{DB}.#{id}.removing" path before 1. and removes it after 2. If there is the path, WAL recovery process removes the object and the path instead of trying recovering the object. --- lib/db.c | 63 +++++++++++++++++++++++++++++++++++++++++- lib/db_wal_recover.cpp | 4 ++- lib/grn_db.h | 2 ++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/lib/db.c b/lib/db.c index 44e186a58d..2c566daebd 100644 --- a/lib/db.c +++ b/lib/db.c @@ -55,8 +55,10 @@ #include "grn_vector.h" #include "grn_index_cursor.h" #include "grn_wal.h" -#include + #include +#include +#include static const uint32_t GRN_TABLE_PAT_KEY_CACHE_SIZE = 1 << 15; @@ -10312,6 +10314,45 @@ grn_obj_remove_log_spec_remove(grn_ctx *ctx, grn_obj_type_to_string(type)); } +static void +grn_ctx_generate_removing_path(grn_ctx *ctx, + grn_id id, + char *path, + size_t path_size) +{ + const char *suffix = ".removing"; + grn_obj_path_by_id(ctx, ctx->impl->db, id, path); + grn_strcat(path, path_size, suffix); +} + +static void +grn_ctx_mark_removing(grn_ctx *ctx, grn_id id) +{ + char path[PATH_MAX]; + grn_ctx_generate_removing_path(ctx, id, path, PATH_MAX); + FILE *removing = grn_fopen(path, "w"); + if (removing) { + fclose(removing); + } +} + +bool +grn_ctx_is_removing(grn_ctx *ctx, grn_id id) +{ + char path[PATH_MAX]; + grn_ctx_generate_removing_path(ctx, id, path, PATH_MAX); + struct stat s; + return stat(path, &s) == 0; +} + +static void +grn_ctx_done_removing(grn_ctx *ctx, grn_id id) +{ + char path[PATH_MAX]; + grn_ctx_generate_removing_path(ctx, id, path, PATH_MAX); + grn_unlink(path); +} + static grn_rc grn_obj_remove_pat(grn_ctx *ctx, grn_obj *obj, @@ -10351,6 +10392,7 @@ grn_obj_remove_pat(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_pat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10360,6 +10402,7 @@ grn_obj_remove_pat(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10408,6 +10451,7 @@ grn_obj_remove_dat(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_dat_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10417,6 +10461,7 @@ grn_obj_remove_dat(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10465,6 +10510,7 @@ grn_obj_remove_hash(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_hash_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10474,6 +10520,7 @@ grn_obj_remove_hash(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10518,6 +10565,7 @@ grn_obj_remove_array(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_array_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10527,6 +10575,7 @@ grn_obj_remove_array(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10560,6 +10609,7 @@ grn_obj_remove_ja(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_ja_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10569,6 +10619,7 @@ grn_obj_remove_ja(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10601,6 +10652,7 @@ grn_obj_remove_ra(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_ra_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10610,6 +10662,7 @@ grn_obj_remove_ra(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10639,6 +10692,7 @@ grn_obj_remove_index(grn_ctx *ctx, } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_ii_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10648,6 +10702,7 @@ grn_obj_remove_index(grn_ctx *ctx, if (rc != GRN_SUCCESS) { return rc; } + grn_ctx_done_removing(ctx, id); } if (!(id & GRN_OBJ_TMP_OBJECT)) { @@ -10672,6 +10727,7 @@ grn_obj_remove_db_obj( } if (path) { + grn_ctx_mark_removing(ctx, id); rc = grn_io_remove(ctx, path); if (rc != GRN_SUCCESS) { return rc; @@ -10687,6 +10743,10 @@ grn_obj_remove_db_obj( grn_obj_touch(ctx, db, NULL); } + if (path) { + grn_ctx_done_removing(ctx, id); + } + return rc; } @@ -10957,6 +11017,7 @@ grn_ctx_remove_internal( grn_strcat(path, PATH_MAX, ".c"); grn_io_remove_if_exist(ctx, path); } + grn_ctx_done_removing(ctx, id); return ctx->rc; } diff --git a/lib/db_wal_recover.cpp b/lib/db_wal_recover.cpp index 5498f951bd..cd926e62db 100644 --- a/lib/db_wal_recover.cpp +++ b/lib/db_wal_recover.cpp @@ -1461,7 +1461,9 @@ grn_db_wal_recover(grn_ctx *ctx, grn_db *db) grn_ctx_push_temporary_open_space(ctx); grn_obj *object = grn_ctx_at(ctx, id); - if (grn_db_wal_recover_is_target_object(ctx, object)) { + if (grn_ctx_is_removing(ctx, id)) { + grn_db_wal_recover_remove_object(ctx, db, object, id); + } else if (grn_db_wal_recover_is_target_object(ctx, object)) { bool is_table = false; bool is_column = false; if (grn_logger_pass(ctx, GRN_LOG_DEBUG)) { diff --git a/lib/grn_db.h b/lib/grn_db.h index 6d4fcb5181..7bac922dc4 100644 --- a/lib/grn_db.h +++ b/lib/grn_db.h @@ -996,6 +996,8 @@ grn_column_create_similar_id_map(grn_ctx *ctx, return column; } +bool grn_ctx_is_removing(grn_ctx *ctx, grn_id id); + #ifdef __cplusplus } #endif From bbfce6824c08daedd21bbae88a1a103a6e0283b0 Mon Sep 17 00:00:00 2001 From: Horimoto Yasuhiro Date: Wed, 3 Jul 2024 13:08:26 +0900 Subject: [PATCH 21/29] db: add missing header for "grn_fopen()" (#1811) GitHub: fix GH-1810 "grn_fopen()" is defined as "_fsopen()" in Windows. We can specify "shflag" in the third argument of "_fsopen()". We use "shflg" in "_fsopen()". We need to include "share.h" to use "shflag". See: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/fsopen-wfsopen?view=msvc-160 However, db.c does not include "share.h". So, "error C2065: '_SH_DENYNO': undeclared identifier" occur. This problem only occur in Groonga for Windows. Because "fopen()" does not have "shflag". ("grn_fopen()" is defined "fopen()" in Linux.) --- lib/db.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/db.c b/lib/db.c index 2c566daebd..2cfcfc6b80 100644 --- a/lib/db.c +++ b/lib/db.c @@ -60,6 +60,10 @@ #include #include +#ifdef WIN32 +# include +#endif + static const uint32_t GRN_TABLE_PAT_KEY_CACHE_SIZE = 1 << 15; #define WITH_NORMALIZE(table, key, key_size, block) \ From 88e9dda1d5fdf5796206f9b383c63f5cbad12436 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 3 Jul 2024 16:13:02 +0900 Subject: [PATCH 22/29] remove_columns_raw: add "." to prefix explicitly GitHub: fix mroonga/mroonga#732 If we don't add ".", we may remove too much objects. For example: Tables: * table1 * table1.column * table12 remove_columns_raw("table1") will remove table1.column and table12. --- lib/db.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/db.c b/lib/db.c index 2cfcfc6b80..a6fcb726da 100644 --- a/lib/db.c +++ b/lib/db.c @@ -9812,24 +9812,17 @@ remove_columns_raw(grn_ctx *ctx, const char *tag = "[table][remove][columns]"; grn_rc rc = GRN_SUCCESS; + char prefix[GRN_TABLE_MAX_KEY_SIZE]; + grn_strncpy(prefix, GRN_TABLE_MAX_KEY_SIZE, table_name, table_name_size); + prefix[table_name_size] = GRN_DB_DELIMITER; GRN_TABLE_EACH_BEGIN_MIN(ctx, ctx->impl->db, cursor, column_id, - /* This prefix includes table itself. We'll - * use table_name as-is and ignore the - * target table instead of appending "." to - * prefix. It's for avoiding memory - * allocation. */ - table_name, - table_name_size, + prefix, + table_name_size + 1 /* GRN_DB_DELIMITER */, GRN_CURSOR_PREFIX) { - /* Ignore the target table. */ - if (column_id == table_id) { - continue; - } - grn_obj *column = grn_ctx_at(ctx, column_id); if (column) { rc = grn_obj_remove_internal(ctx, column, flags); From bb4a1bd4cd11e374bd165bc8232c6e6f0f54f032 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 3 Jul 2024 16:20:46 +0900 Subject: [PATCH 23/29] test table_remove: add a test for 88e9dda1d5fdf5796206f9b383c63f5cbad12436 --- .../common_prefix_table_name.expected | 19 +++++++++++++++++++ .../common_prefix_table_name.test | 12 ++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 test/command/suite/table_remove/patricia_trie/common_prefix_table_name.expected create mode 100644 test/command/suite/table_remove/patricia_trie/common_prefix_table_name.test diff --git a/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.expected b/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.expected new file mode 100644 index 0000000000..9e93669699 --- /dev/null +++ b/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.expected @@ -0,0 +1,19 @@ +table_create Users TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +column_create Users name COLUMN_SCALAR ShortText +[[0,0.0,0.0],true] +table_create Users2 TABLE_PAT_KEY ShortText +[[0,0.0,0.0],true] +dump +table_create Users TABLE_PAT_KEY ShortText +column_create Users name COLUMN_SCALAR ShortText + +table_create Users2 TABLE_PAT_KEY ShortText +table_remove Users +[[0,0.0,0.0],true] +#|n| DDL:256:obj_remove Users +#|n| DDL:257:obj_remove Users.name +#|n| spec:257:remove:Users.name:65(column:var_size) +#|n| spec:256:remove:Users:49(table:pat_key) +dump +table_create Users2 TABLE_PAT_KEY ShortText diff --git a/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.test b/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.test new file mode 100644 index 0000000000..fbdff8df6e --- /dev/null +++ b/test/command/suite/table_remove/patricia_trie/common_prefix_table_name.test @@ -0,0 +1,12 @@ +table_create Users TABLE_PAT_KEY ShortText +column_create Users name COLUMN_SCALAR ShortText +table_create Users2 TABLE_PAT_KEY ShortText + +dump + +#@add-important-log-levels notice +table_remove Users +#@remove-important-log-levels notice + +#@read-timeout 0.1 +dump From d9cb3becbabde715a7f20d46216af242cff998c6 Mon Sep 17 00:00:00 2001 From: Horimoto Yasuhiro Date: Wed, 3 Jul 2024 18:34:11 +0900 Subject: [PATCH 24/29] doc news: add missing contents (#1812) --- doc/locale/ja/LC_MESSAGES/news/14.po | 25 ++++++++++++++++++++----- doc/source/news/14.md | 16 +++++++++++++++- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/doc/locale/ja/LC_MESSAGES/news/14.po b/doc/locale/ja/LC_MESSAGES/news/14.po index a639e0c387..1f2a27380d 100644 --- a/doc/locale/ja/LC_MESSAGES/news/14.po +++ b/doc/locale/ja/LC_MESSAGES/news/14.po @@ -17,8 +17,26 @@ msgstr "" msgid "News - 14 series" msgstr "お知らせ - 14系" -msgid "Release 14.0.5 - 2024-06-29" -msgstr "14.0.5リリース - 2024-06-29" +msgid "Release 14.0.5 - 2024-07-04" +msgstr "14.0.5リリース - 2024-07-04" + +msgid "Improvements" +msgstr "改良" + +msgid "Dropped support for CentOS 7." +msgstr "CentOS 7のサポートをやめました。" + +msgid "Because CentOS 7 reached EOL." +msgstr "CentOS 7はEOLになったためです。" + +msgid "Added a new feature that objects(table or column) as remove as possible." +msgstr "可能な限りオブジェクト（テーブルやカラム）を削除する機能を追加しました。" + +msgid "The crash safe feature of PGroonga will use this feature mainly." +msgstr "この機能は主にPGroongaのクラッシュセーフ機能で使います。" + +msgid "PGroonga will apply PGroonga's WAL to standby database automatically by using [`Custom WAL Resource Managers`](https://www.postgresql.org/docs/current/custom-rmgr.html). However, when PGroonga use `Custom WAL Resource Managers`, all replications are stop if PGroonga fail application of PGroonga's WAL due to break Groonga's object. So, if broken objects exist in database, Groonga will try as remove as possible objects by using this feature." +msgstr "PGroongaは[`Custom WAL Resource Managers`](https://www.postgresql.jp/document/16/html/custom-rmgr.html)を使うことで、PGroongaのWALをスタンバイへ自動で適用できるようになります。ただ、`Custom WAL Resource Managers`を使用していると、オブジェクトの破損などでPGroongaのWALの適用に失敗した場合にすべてのレプリケーションが停止します。そのため、壊れたオブジェクトがデータベース内に存在する場合、Groongaはこの機能を使って、できる限り壊れたオブジェクトを削除しようとします。" msgid "Fixes" msgstr "修正" @@ -83,9 +101,6 @@ msgstr "以下の例では、Groongaが`[1, \"Linux MySQL\"]` と `[2, \"MySQL G msgid "Release 14.0.3 - 2024-05-09" msgstr "14.0.3リリース - 2024-05-09" -msgid "Improvements" -msgstr "改良" - msgid "We optimized performance as below." msgstr "以下の最適化を行いました。" diff --git a/doc/source/news/14.md b/doc/source/news/14.md index d1aeeec88b..e2f752c9a7 100644 --- a/doc/source/news/14.md +++ b/doc/source/news/14.md @@ -1,7 +1,21 @@ # News - 14 series (release-14-0-5)= -## Release 14.0.5 - 2024-06-29 +## Release 14.0.5 - 2024-07-04 + +### Improvements + +* Dropped support for CentOS 7. + + Because CentOS 7 reached EOL. + +* Added a new feature that objects(table or column) as remove as possible. + + The crash safe feature of PGroonga will use this feature mainly. + + PGroonga will apply PGroonga's WAL to standby database automatically by using [`Custom WAL Resource Managers`](https://www.postgresql.org/docs/current/custom-rmgr.html). + However, when PGroonga use `Custom WAL Resource Managers`, all replications are stop if PGroonga fail application of PGroonga's WAL due to break Groonga's object. + So, if broken objects exist in database, Groonga will try as remove as possible objects by using this feature. ### Fixes From e52e5b28b514c68c0984ba6b4395c41f1b5c6b96 Mon Sep 17 00:00:00 2001 From: otegami Date: Fri, 26 Apr 2024 20:43:18 +0800 Subject: [PATCH 25/29] Revert "ci macos: add workaround for msgpack-c" This reverts commit 8275463f7d8f8fc7308b232049fdb0cf538bd3d5. This problem is fixed by the following commit. - ref: https://github.com/msgpack/msgpack-c/commit/68cc50a3de991e5b534ac628b7f9d1a0475e5bfc --- .github/workflows/autotools.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index 66e6bb381d..e3426661d7 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -196,13 +196,6 @@ jobs: path: ccache key: autotools-macos-ccache-${{ hashFiles('lib/**', 'src/**', 'plugins/**', 'include/**') }} restore-keys: autotools-macos-ccache- - - name: Adjust msgpack-c.pc - run: | - sed \ - -i.bak \ - -E \ - -e 's,^(include|lib)dir=,\1dir=${prefix}/,g' \ - $(brew --prefix msgpack)/lib/pkgconfig/msgpack-c.pc - name: Generate configure run: | ./autogen.sh From 720ea257cea6b68dcedbbf4682db5b65055d07d6 Mon Sep 17 00:00:00 2001 From: otegami Date: Thu, 13 Jun 2024 20:55:31 +0800 Subject: [PATCH 26/29] skip linux --- .github/workflows/autotools.yml | 236 ++++++++++++++++---------------- 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index e3426661d7..f1a350ff84 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -55,125 +55,125 @@ concurrency: group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: - linux: - name: Linux - strategy: - fail-fast: false - matrix: - include: - - cc: gcc - cxx: g++ - - cc: clang - cxx: clang++ - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Prepare ccache - run: | - echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ccache - key: autotools-linux-${{ matrix.cc }}-ccache-${{ hashFiles('lib/**', 'src/**', 'plugins/**', 'include/**') }} - restore-keys: autotools-linux-${{ matrix.cc }}-ccache- - - name: Enable Apache Arrow repository - run: | - sudo apt update -o="APT::Acquire::Retries=3" - sudo apt install -y -V -o="APT::Acquire::Retries=3" \ - lsb-release \ - wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - - name: Install packages - run: | - sudo apt update -o="APT::Acquire::Retries=3" - sudo apt install -y -V -o="APT::Acquire::Retries=3" \ - autoconf-archive \ - automake1.11 \ - autotools-dev \ - ccache \ - clang \ - gdb \ - gettext \ - libarrow-dev \ - libevent-dev \ - libluajit-5.1-dev \ - liblz4-dev \ - libmecab-dev \ - libmsgpack-dev \ - libstemmer-dev \ - libxxhash-dev \ - libzstd-dev \ - mecab-naist-jdic \ - rapidjson-dev \ - ruby-dev \ - zlib1g-dev - echo "/usr/lib/ccache" >> $GITHUB_PATH - - name: Install Cutter - run: | - curl \ - --silent \ - --location \ - https://raw.github.com/clear-code/cutter/master/data/travis/setup.sh | sh - - name: Generate configure - run: | - ./autogen.sh - - name: Configure - run: | - ccache --show-stats --verbose --version || : - ./configure \ - CC=${{ matrix.cc }} \ - CXX=${{ matrix.cxx }} \ - --enable-debug \ - --enable-mruby \ - --prefix=$PWD/install \ - --with-ruby - - name: Build - run: | - make -j$(nproc) - ccache --show-stats --verbose --version || : - - name: Install - run: | - make install - - name: Install test dependencies - run: | - sudo env MAKEFLAGS=-j$(nproc) gem install \ - grntest \ - pkg-config \ - rroonga - - name: Set environment variables - run: | - echo "COLUMNS=79" >> ${GITHUB_ENV} - echo "LD_LIBRARY_PATH=$PWD/install/lib" >> ${GITHUB_ENV} - echo "TZ=Asia/Tokyo" >> ${GITHUB_ENV} + # linux: + # name: Linux + # strategy: + # fail-fast: false + # matrix: + # include: + # - cc: gcc + # cxx: g++ + # - cc: clang + # cxx: clang++ + # runs-on: ubuntu-latest + # timeout-minutes: 45 + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: recursive + # - name: Prepare ccache + # run: | + # echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + # - name: Cache ccache + # uses: actions/cache@v4 + # with: + # path: ccache + # key: autotools-linux-${{ matrix.cc }}-ccache-${{ hashFiles('lib/**', 'src/**', 'plugins/**', 'include/**') }} + # restore-keys: autotools-linux-${{ matrix.cc }}-ccache- + # - name: Enable Apache Arrow repository + # run: | + # sudo apt update -o="APT::Acquire::Retries=3" + # sudo apt install -y -V -o="APT::Acquire::Retries=3" \ + # lsb-release \ + # wget + # wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + # sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + # - name: Install packages + # run: | + # sudo apt update -o="APT::Acquire::Retries=3" + # sudo apt install -y -V -o="APT::Acquire::Retries=3" \ + # autoconf-archive \ + # automake1.11 \ + # autotools-dev \ + # ccache \ + # clang \ + # gdb \ + # gettext \ + # libarrow-dev \ + # libevent-dev \ + # libluajit-5.1-dev \ + # liblz4-dev \ + # libmecab-dev \ + # libmsgpack-dev \ + # libstemmer-dev \ + # libxxhash-dev \ + # libzstd-dev \ + # mecab-naist-jdic \ + # rapidjson-dev \ + # ruby-dev \ + # zlib1g-dev + # echo "/usr/lib/ccache" >> $GITHUB_PATH + # - name: Install Cutter + # run: | + # curl \ + # --silent \ + # --location \ + # https://raw.github.com/clear-code/cutter/master/data/travis/setup.sh | sh + # - name: Generate configure + # run: | + # ./autogen.sh + # - name: Configure + # run: | + # ccache --show-stats --verbose --version || : + # ./configure \ + # CC=${{ matrix.cc }} \ + # CXX=${{ matrix.cxx }} \ + # --enable-debug \ + # --enable-mruby \ + # --prefix=$PWD/install \ + # --with-ruby + # - name: Build + # run: | + # make -j$(nproc) + # ccache --show-stats --verbose --version || : + # - name: Install + # run: | + # make install + # - name: Install test dependencies + # run: | + # sudo env MAKEFLAGS=-j$(nproc) gem install \ + # grntest \ + # pkg-config \ + # rroonga + # - name: Set environment variables + # run: | + # echo "COLUMNS=79" >> ${GITHUB_ENV} + # echo "LD_LIBRARY_PATH=$PWD/install/lib" >> ${GITHUB_ENV} + # echo "TZ=Asia/Tokyo" >> ${GITHUB_ENV} - echo "$PWD/install/bin" >> ${GITHUB_PATH} - - name: "Test: API" - run: | - bash -x test/unit/run-test.sh - - name: "Test: mruby" - run: | - USE_SYSTEM=yes test/mruby/run-test.rb - - uses: actions/cache@v4 - with: - path: | - ~/.cache/red-datasets - key: red-datasets-ubuntu - - name: "Test: command line" - run: | - test/command_line/run-test.rb --groonga-install-prefix=$PWD/install - - name: "Test: stdio" - run: | - grntest \ - --base-dir test/command \ - --n-retries=2 \ - --read-timeout=30 \ - --reporter=mark \ - test/command/suite + # echo "$PWD/install/bin" >> ${GITHUB_PATH} + # - name: "Test: API" + # run: | + # bash -x test/unit/run-test.sh + # - name: "Test: mruby" + # run: | + # USE_SYSTEM=yes test/mruby/run-test.rb + # - uses: actions/cache@v4 + # with: + # path: | + # ~/.cache/red-datasets + # key: red-datasets-ubuntu + # - name: "Test: command line" + # run: | + # test/command_line/run-test.rb --groonga-install-prefix=$PWD/install + # - name: "Test: stdio" + # run: | + # grntest \ + # --base-dir test/command \ + # --n-retries=2 \ + # --read-timeout=30 \ + # --reporter=mark \ + # test/command/suite macos: name: macOS From 9632de3fc418fd011d1f7952bf6b454bdb8a62f1 Mon Sep 17 00:00:00 2001 From: otegami Date: Thu, 13 Jun 2024 20:45:33 +0800 Subject: [PATCH 27/29] Use hotfixed msgpack-c --- .github/workflows/autotools.yml | 11 +++++++++++ Brewfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index f1a350ff84..dcd34d2fd5 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -186,6 +186,17 @@ jobs: - name: Install packages run: | brew bundle + - uses: actions/checkout@v4 + with: + path: msgpack-c + ref: fix-ignored-includedir + repository: otegami/msgpack-c + - name: Build msgpack-c + run: | + cd msgpack-c + cmake -S . -B ../msgpack-c.build -DCMAKE_INSTALL_PREFIX=/usr/local + cmake --build ../msgpack-c.build + sudo cmake --install ../msgpack-c.build - name: Prepare ccache run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} diff --git a/Brewfile b/Brewfile index 8bb52c62b9..333ab67d7a 100644 --- a/Brewfile +++ b/Brewfile @@ -10,7 +10,7 @@ brew "libevent" brew "libtool" brew "mecab" brew "mecab-ipadic" -brew "msgpack" +# brew "msgpack" brew "ninja" brew "openssl" brew "pcre" From 50d9fb592a82bf547e6a6b0bac35afc2242b6b73 Mon Sep 17 00:00:00 2001 From: otegami Date: Fri, 21 Jun 2024 20:38:56 +0800 Subject: [PATCH 28/29] Use CMAKE_INSTALL_*DIRs --- .github/workflows/autotools.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index dcd34d2fd5..24213874de 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -194,7 +194,9 @@ jobs: - name: Build msgpack-c run: | cd msgpack-c - cmake -S . -B ../msgpack-c.build -DCMAKE_INSTALL_PREFIX=/usr/local + cmake -S . -B ../msgpack-c.build -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_INSTALL_INCLUDEDIR=include cmake --build ../msgpack-c.build sudo cmake --install ../msgpack-c.build - name: Prepare ccache From 92753bbe9868431402592263642f708e0ba23848 Mon Sep 17 00:00:00 2001 From: otegami Date: Mon, 24 Jun 2024 20:09:06 +0800 Subject: [PATCH 29/29] Try to use the new release version --- .github/workflows/autotools.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml index 24213874de..f29b72c2fb 100644 --- a/.github/workflows/autotools.yml +++ b/.github/workflows/autotools.yml @@ -189,8 +189,8 @@ jobs: - uses: actions/checkout@v4 with: path: msgpack-c - ref: fix-ignored-includedir - repository: otegami/msgpack-c + ref: c-6.0.2 + repository: msgpack/msgpack-c - name: Build msgpack-c run: | cd msgpack-c