diff --git a/db/osqlcomm.c b/db/osqlcomm.c index ae40ac5955..d8b6f18660 100644 --- a/db/osqlcomm.c +++ b/db/osqlcomm.c @@ -6687,6 +6687,135 @@ static struct schema_change_type* _create_logical_cron_systable(const char *tbln return sc; } +/** + * Change the retention of an existing truncate (TIME or MANUAL) partition. + * Reconfigures the ring in memory (see timepart_reconfigure_retention), then + * chains one schema change per shard that must be physically created + * (increase, SC_ADDTABLE, mirroring _process_single_table_sc_partitioning) or + * dropped (decrease, SC_DROPTABLE, mirroring the per-shard drop shape used by + * partition merge). The first chained schema change carries sc->newpartition + * and publish/unpublish, so the reconfigured view is swapped in and llmeta + * updated atomically with the whole chain, once all shards finalize. + */ +static int _process_partition_retention(struct ireq *iq) +{ + struct schema_change_type *sc = iq->sc; + struct errstat err = {0}; + timepart_view_t *newview = NULL; + const char *partition_name = NULL; + char **names = NULL; + int nnames = 0; + int is_increase = 0; + int rc; + int i; + + assert(sc->kind == SC_ALTERTABLE); + + rc = timepart_reconfigure_retention(sc->tablename, sc->partition.u.tpt.retention, + sc->partition.u.tpt.period == VIEW_PARTITION_MANUAL, &newview, &partition_name, + &names, &nnames, &is_increase, &err); + if (rc) { + logmsg(LOGMSG_ERROR, "Failed to change retention for %s rc %d \"%s\"\n", sc->tablename, err.errval, err.errstr); + sc_errf(sc, "Failed to change retention for %s: %s", sc->tablename, err.errstr); + return ERR_SC; + } + + if (nnames == 0) { + /* retention unchanged, nothing to do */ + if (newview) + timepart_free_view(newview); + return SC_OK; + } + + sc->timepartition_name = partition_name; + sc->force_rebuild = 0; /* no data movement */ + sc->nothrevent = 1; /* serialize, mirrors create/rollout paths */ + + rc = SC_OK; + for (i = 0; i < nnames; i++) { + struct schema_change_type *cur; + + if (i == 0) { + cur = sc; + } else { + cur = clone_schemachange_type(sc); + if (!cur) { + rc = ERR_SC; + break; + } + } + cur->iq = iq; + cur->tran = sc->tran; + cur->finalize = 0; /* make sure */ + strncpy0(cur->tablename, names[i], sizeof(cur->tablename)); + + if (is_increase) { + cur->kind = SC_ADDTABLE; + } else { + char *schemabuf = NULL; + + cur->kind = SC_DROPTABLE; + cur->same_schema = 1; + free(cur->newcsc2); + cur->newcsc2 = NULL; + if (get_csc2_file(names[i], -1, &schemabuf, NULL)) { + sc_errf(cur, "could not get schema for shard '%s'", names[i]); + if (cur != sc) + free_schema_change_type(cur); + rc = ERR_SC; + break; + } + cur->newcsc2 = schemabuf; + } + + iq->sc = cur; + rc = start_schema_change_tran(iq, NULL); + if (rc != SC_OK || cur->preempted == SC_ACTION_RESUME || cur->kind == SC_ALTERTABLE_PENDING) { + iq->sc = NULL; + /* link even on failure so backout frees cur and clears any running + * state it registered (matches start_schema_change_tran_wrapper) */ + if (cur->nothrevent) { + cur->sc_next = iq->sc_pending; + iq->sc_pending = cur; + } else if (cur != sc) { + free_schema_change_type(cur); + } + if (rc != SC_OK) { + if (rc != SC_MASTER_DOWNGRADE) + iq->osql_flags |= OSQL_FLAGS_SCDONE; + else + iq->osql_flags &= ~OSQL_FLAGS_SCDONE; + rc = ERR_SC; + } + break; + } + iq->sc->sc_next = iq->sc_pending; + iq->sc_pending = iq->sc; + } + + if (rc == SC_OK) { + /* Attach the reconfigured view + publish/unpublish to the first shard + * sc. It is at the tail of sc_pending (LIFO) so it finalizes last, + * after every shard has been created/dropped, swapping the in-memory + * view and rewriting llmeta atomically with the whole chain. + * Deferred to here so newview is owned by an sc only once the chain is + * fully built -- otherwise (early failure below) we free it ourselves. */ + sc->newpartition = newview; + sc->publish = partition_publish; + sc->unpublish = partition_unpublish; + iq->osql_flags |= OSQL_FLAGS_SCDONE; + } else { + /* nothing was published; no sc owns newview, so free it here */ + timepart_free_view(newview); + } + + for (i = 0; i < nnames; i++) + free(names[i]); + free(names); + + return rc; +} + static int _process_partition_alter_and_drop(struct ireq *iq) { struct schema_change_type *sc = iq->sc; @@ -6717,6 +6846,10 @@ static int _process_partition_alter_and_drop(struct ireq *iq) return _process_partitioned_table_merge(iq); } + if (sc->partition.type == PARTITION_RETENTION) { + return _process_partition_retention(iq); + } + timepart_sc_arg_t arg = {0}; arg.s = sc; arg.s->iq = iq; diff --git a/db/views.c b/db/views.c index bdb40d6294..7695d720b4 100644 --- a/db/views.c +++ b/db/views.c @@ -2701,6 +2701,241 @@ int timepart_update_retention(void *tran, const char *name, int retention, struc return rc; } +/** + * Build a reconfigured view reflecting a new retention for a truncate + * (TIME or MANUAL) partition, without mutating the live view. + * + * The shards are walked in rollout order starting right after the current + * shard (i.e. oldest-next-to-be-recycled first), ending with the current + * shard itself. The reconfigured ring is built so that the (unchanged) + * current shard always ends up at the last index: + * - increase: new empty shards are prepended (consumed next, preserving + * history), followed by the old shards in rollout order. + * - decrease: the oldest shards are dropped from the front of the rollout + * order, keeping the remaining ones (still ending with current). + * Returns *newview (caller frees via timepart_free_view) and the list of + * shard table names to physically create (increase) or drop (decrease). + */ +int timepart_reconfigure_retention(const char *name, int new_retention, int is_manual, timepart_view_t **newview_out, + const char **partition_name_out, char ***names_out, int *nnames_out, + int *is_increase_out, struct errstat *err) +{ + timepart_view_t *view; + timepart_view_t *newview = NULL; + timepart_shard_t *old_order = NULL; + char **names = NULL; + int old_retention; + int is_increase = 0; + int nnames = 0; + int i; + int rc = VIEW_NOERR; + + *newview_out = NULL; + *partition_name_out = NULL; + *names_out = NULL; + *nnames_out = 0; + *is_increase_out = 0; + + Pthread_rwlock_rdlock(&views_lk); + + view = _get_view(thedb->timepart_views, name); + if (!view) { + errstat_set_strf(err, "Partition %s doesn't exist!", name); + errstat_set_rc(err, rc = VIEW_ERR_EXIST); + goto done_locked; + } + + if (view->rolltype != TIMEPART_ROLLOUT_TRUNCATE) { + errstat_set_strf(err, "Partition %s is a legacy partition, use PUT instead", name); + errstat_set_rc(err, rc = VIEW_ERR_PARAM); + goto done_locked; + } + + if ((view->period == VIEW_PARTITION_MANUAL) != (is_manual != 0)) { + errstat_set_strf(err, "Partition %s is %s, use PARTITIONED BY %s", name, + view->period == VIEW_PARTITION_MANUAL ? "manual" : "time", + view->period == VIEW_PARTITION_MANUAL ? "MANUAL" : "TIME"); + errstat_set_rc(err, rc = VIEW_ERR_PARAM); + goto done_locked; + } + + old_retention = view->retention; + if (new_retention == old_retention) + goto done_locked; /* no-op, nnames stays 0 */ + + if (new_retention >= VIEWS_MAX_RETENTION) { + errstat_set_strf(err, "Retention too high for \"%s\"", name); + errstat_set_rc(err, rc = VIEW_ERR_PARAM); + goto done_locked; + } + + is_increase = new_retention > old_retention; + + /* snapshot the old shards in rollout order (oldest-next .. current); + * these tblname pointers are borrowed from the live view, not owned */ + old_order = malloc(sizeof(timepart_shard_t) * view->nshards); + if (!old_order) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + for (i = 0; i < view->nshards; i++) { + old_order[i] = view->shards[(view->current_shard + 1 + i) % view->nshards]; + } + + newview = (timepart_view_t *)calloc(1, sizeof(timepart_view_t)); + if (!newview) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + newview->name = strdup(view->name); + newview->shard0name = strdup(view->shard0name); + if (!newview->name || !newview->shard0name) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + newview->period = view->period; + newview->starttime = view->starttime; + newview->roll_time = view->roll_time; + comdb2uuidcpy(newview->source_id, view->source_id); + newview->rolltype = TIMEPART_ROLLOUT_TRUNCATE; + newview->retention = new_retention; + newview->nshards = new_retention; + newview->shards = (timepart_shard_t *)calloc(new_retention, sizeof(timepart_shard_t)); + if (!newview->shards) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + + if (is_increase) { + int nnew = new_retention - old_retention; + char *prev_name = view->shards[view->current_shard].tblname; + + names = calloc(nnew, sizeof(char *)); + if (!names) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + nnames = nnew; /* so the error path below frees any partial names[] */ + for (i = 0; i < nnew; i++) { + char newname[MAXTABLELEN + 1]; + + rc = _generate_new_shard_name(prev_name, newname, sizeof(newname), old_retention + i, new_retention, + view->period == VIEW_PARTITION_TEST2MIN, err); + if (rc != VIEW_NOERR) + goto done_locked; + if (get_dbtable_by_name(newname)) { + errstat_set_rcstrf(err, rc = VIEW_ERR_EXIST, "shard %s exists", newname); + goto done_locked; + } + newview->shards[i].tblname = strdup(newname); + names[i] = strdup(newname); + if (!newview->shards[i].tblname || !names[i]) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + newview->shards[i].low = INT_MAX; + newview->shards[i].high = INT_MAX; + prev_name = newname; + } + /* old shards, in rollout order, follow the new empties; current + * shard (old_order[old_retention-1]) lands last */ + for (i = 0; i < old_retention; i++) { + newview->shards[nnew + i].tblname = strdup(old_order[i].tblname); + if (!newview->shards[nnew + i].tblname) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + newview->shards[nnew + i].low = old_order[i].low; + newview->shards[nnew + i].high = old_order[i].high; + } + newview->current_shard = new_retention - 1; + nnames = nnew; + } else { + int ndrop = old_retention - new_retention; + + names = calloc(ndrop, sizeof(char *)); + if (!names) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + nnames = ndrop; /* so the error path below frees any partial names[] */ + for (i = 0; i < ndrop; i++) { + names[i] = strdup(old_order[i].tblname); + if (!names[i]) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + } + for (i = 0; i < new_retention; i++) { + newview->shards[i].tblname = strdup(old_order[ndrop + i].tblname); + if (!newview->shards[i].tblname) { + errstat_set_rcstrf(err, rc = VIEW_ERR_MALLOC, "malloc %s %d", __func__, __LINE__); + goto done_locked; + } + newview->shards[i].low = old_order[ndrop + i].low; + newview->shards[i].high = old_order[ndrop + i].high; + } + /* the dropped shards' data is discarded; the surviving shards keep + * their existing [low,high) coverage unchanged */ + newview->current_shard = new_retention - 1; + nnames = ndrop; + } + + rc = VIEW_NOERR; + +done_locked: + Pthread_rwlock_unlock(&views_lk); + + free(old_order); + + if (rc != VIEW_NOERR) { + if (newview) { + timepart_free_view(newview); + newview = NULL; + } + if (names) { + for (i = 0; i < nnames; i++) + free(names[i]); + free(names); + names = NULL; + } + return rc; + } + + *newview_out = newview; + if (newview) + *partition_name_out = newview->name; + *names_out = names; + *nnames_out = nnames; + *is_increase_out = is_increase; + return VIEW_NOERR; +} + +/** + * Swap an existing partition's in-memory view for a reconfigured one with + * the same name/source_id (used after a retention change). Pending cron + * events reference the partition by name and re-fetch the view when they + * fire, so this does not disturb rollout scheduling. + */ +int timepart_replace_inmem_view(timepart_view_t *newview) +{ + timepart_views_t *views = thedb->timepart_views; + timepart_view_t *oldview; + int idx = -1; + + Pthread_rwlock_wrlock(&views_lk); + oldview = _get_view_index(views, newview->name, &idx); + if (!oldview) { + Pthread_rwlock_unlock(&views_lk); + return VIEW_ERR_EXIST; + } + views->views[idx] = newview; + Pthread_rwlock_unlock(&views_lk); + + timepart_free_view(oldview); + return VIEW_NOERR; +} + /** * Locking the views subsystem, needed for ordering locks with schema * @@ -3429,6 +3664,16 @@ int partition_publish(tran_type *tran, struct schema_change_type *sc) abort(); /* restart will fix this*/ break; } + case PARTITION_RETENTION: { + assert(sc->newpartition != NULL); + rc = timepart_replace_inmem_view(sc->newpartition); + if (rc) + abort(); /* restart will fix this*/ + /* ownership transferred to the views list; clear so backout does + * not free the now-live view */ + sc->newpartition = NULL; + break; + } } /*switch */ int bdberr = 0; rc = bdb_llog_partition(thedb->bdb_env, tran, @@ -3463,6 +3708,17 @@ void partition_unpublish(struct schema_change_type *sc) abort(); /* restart will fix this*/ break; } + case PARTITION_RETENTION: { + /* the pre-change view was already freed by a successful publish + * swap; a later sc's publish failing after this leaves us unable + * to cleanly revert in memory, restart will reload from llmeta */ + logmsg(LOGMSG_ERROR, + "%s: cannot revert retention change for %s in memory, " + "restart required\n", + __func__, sc->timepartition_name); + abort(); + break; + } } } } @@ -3509,7 +3765,7 @@ int timepart_rollout(const char *partname) timepart_view_t *view = _get_view(thedb->timepart_views, partname); if (!view) { goto done; - } + } if (view->rolltype == TIMEPART_ROLLOUT_TRUNCATE) { rc = ROLLOUT_TRUNC; @@ -3522,6 +3778,19 @@ int timepart_rollout(const char *partname) return rc; } +int timepart_get_period(const char *partname) +{ + int period = VIEW_PARTITION_INVALID; + + Pthread_rwlock_rdlock(&views_lk); + timepart_view_t *view = _get_view(thedb->timepart_views, partname); + if (view) + period = view->period; + Pthread_rwlock_unlock(&views_lk); + + return period; +} + #ifdef COMDB2_TEST __thread int have_views_lk = 0; #endif diff --git a/db/views.h b/db/views.h index ccf6b4d507..6b58cfd970 100644 --- a/db/views.h +++ b/db/views.h @@ -340,6 +340,25 @@ int views_cron_restart(timepart_views_t *views); */ int timepart_update_retention(void *tran, const char *name, int value, struct errstat *err); +/** + * Build a new view reflecting a retention change for a truncate partition. + * Produces *newview (the reconfigured ring, current shard at the last index) + * and the list of shard table names that must be physically added (increase) + * or dropped (decrease); *is_increase is set accordingly. If the retention is + * unchanged, newview and names are set to NULL and nnames is 0. + * The caller owns *newview (timepart_free_view) and the *names array (free + * each string then the array). + */ +int timepart_reconfigure_retention(const char *name, int new_retention, int is_manual, struct timepart_view **newview, + const char **partition_name, char ***names, int *nnames, int *is_increase, + struct errstat *err); + +/** + * Swap an existing partition's in-memory view for a reconfigured one with + * the same name (used after a retention change publishes). + */ +int timepart_replace_inmem_view(struct timepart_view *newview); + /** * Locking the views subsystem, needed for ordering locks with schema * @@ -549,6 +568,12 @@ enum { */ int timepart_rollout(const char *partname); +/** + * Return the rollout period (enum view_partition_period) of an existing + * partition, or VIEW_PARTITION_INVALID if the partition does not exist. + */ +int timepart_get_period(const char *partname); + /** * Analyze all shards of a "name" partition * diff --git a/schemachange/sc_add_table.c b/schemachange/sc_add_table.c index feba798790..1aa706778b 100644 --- a/schemachange/sc_add_table.c +++ b/schemachange/sc_add_table.c @@ -367,8 +367,19 @@ int finalize_add_table(struct ireq *iq, struct schema_change_type *s, * of merging another table in, in which case tablename is provided) * Done only from one shard, the one that will publish results */ - } else if (s->partition.type == PARTITION_MERGE && - s->partition.u.mergetable.tablename[0] == '\0') { + } else if (s->partition.type == PARTITION_RETENTION && s->publish) { + /* increasing retention: persist the reconfigured (larger) ring after + * the last new empty shard has been created */ + struct errstat err = {0}; + assert(s->newpartition); + rc = partition_llmeta_write(tran, s->newpartition, 1, &err); + if (rc) { + logmsg(LOGMSG_ERROR, "Failed to update partition %s retention rc %d \"%s\"\n", s->timepartition_name, rc, + err.errstr); + sc_errf(s, "partition_llmeta_write failed \"retention\"\n"); + return -1; + } + } else if (s->partition.type == PARTITION_MERGE && s->partition.u.mergetable.tablename[0] == '\0') { struct errstat err = {0}; if (partition_llmeta_delete(tran, s->timepartition_name, &err)) { sc_errf(s, "Failed to remove partition llmeta %d\n", err.errval); diff --git a/schemachange/sc_drop_table.c b/schemachange/sc_drop_table.c index 7faa6eba01..d88e1f4bb1 100644 --- a/schemachange/sc_drop_table.c +++ b/schemachange/sc_drop_table.c @@ -141,6 +141,16 @@ int finalize_drop_table(struct ireq *iq, struct schema_change_type *s, sc_errf(s, "Failed to remove partition llmeta %d\n", err.errval); return SC_INTERNAL_ERROR; } + } else if (s->partition.type == PARTITION_RETENTION && s->publish) { + /* decreasing retention: persist the reconfigured (smaller) ring + * after the last oldest shard has been dropped */ + struct errstat err = {0}; + assert(s->newpartition); + rc = partition_llmeta_write(tran, s->newpartition, 1, &err); + if (rc) { + sc_errf(s, "Failed to update partition %s retention rc %d\n", s->timepartition_name, err.errval); + return SC_INTERNAL_ERROR; + } } else if (s->partition.type == PARTITION_REM_GENSHARD) { struct errstat err = {0}; rc = gen_shard_llmeta_remove(tran, db->sqlaliasname, &err); diff --git a/schemachange/sc_logic.c b/schemachange/sc_logic.c index 8a12f1b6be..edfcb6c568 100644 --- a/schemachange/sc_logic.c +++ b/schemachange/sc_logic.c @@ -1781,6 +1781,13 @@ int backout_schema_changes(struct ireq *iq, tran_type *tran) } } /* TODO: (NC) Also delete view? */ + /* a retention change carries a reconfigured view that publish would + * have handed to the views list; since we are backing out (publish did + * not run, or it cleared newpartition on success), free it here */ + if (s->partition.type == PARTITION_RETENTION && s->newpartition) { + timepart_free_view(s->newpartition); + s->newpartition = NULL; + } sc_del_unused_files_tran(s->db, tran); s = iq->sc = s->sc_next; } diff --git a/schemachange/sc_struct.c b/schemachange/sc_struct.c index 70f9656517..00c81f7b4f 100644 --- a/schemachange/sc_struct.c +++ b/schemachange/sc_struct.c @@ -183,6 +183,7 @@ static size_t _partition_packed_size(struct comdb2_partition *p) case PARTITION_ADD_TIMED: case PARTITION_ADD_TIMED_RETRO: case PARTITION_ADD_MANUAL: + case PARTITION_RETENTION: return sizeof(p->type) + sizeof(p->u.tpt.period) + sizeof(p->u.tpt.retention) + sizeof(p->u.tpt.start); case PARTITION_MERGE: @@ -349,7 +350,8 @@ int pack_schema_change_protobuf(struct schema_change_type *s, void **packed_sc, switch (s->partition.type) { case PARTITION_ADD_TIMED: case PARTITION_ADD_TIMED_RETRO: - case PARTITION_ADD_MANUAL: { + case PARTITION_ADD_MANUAL: + case PARTITION_RETENTION: { sc.has_tpperiod = 1; sc.has_tpretention = 1; sc.has_tpstart = 1; @@ -668,7 +670,8 @@ int unpack_schema_change_protobuf(struct schema_change_type *s, void *packed_sc, switch (sc->partition_type) { case PARTITION_ADD_TIMED: case PARTITION_ADD_TIMED_RETRO: - case PARTITION_ADD_MANUAL: { + case PARTITION_ADD_MANUAL: + case PARTITION_RETENTION: { s->partition.u.tpt.period = sc->tpperiod; s->partition.u.tpt.retention = sc->tpretention; s->partition.u.tpt.start = sc->tpstart; @@ -856,7 +859,8 @@ void *buf_put_schemachange(struct schema_change_type *s, void *p_buf, void *p_bu p_buf_end); switch (s->partition.type) { case PARTITION_ADD_TIMED: - case PARTITION_ADD_MANUAL: { + case PARTITION_ADD_MANUAL: + case PARTITION_RETENTION: { p_buf = buf_put(&s->partition.u.tpt.period, sizeof(s->partition.u.tpt.period), p_buf, p_buf_end); p_buf = buf_put(&s->partition.u.tpt.retention, @@ -1301,7 +1305,8 @@ void *buf_get_schemachange_v2(struct schema_change_type *s, switch (s->partition.type) { case PARTITION_ADD_TIMED: case PARTITION_ADD_TIMED_RETRO: - case PARTITION_ADD_MANUAL: { + case PARTITION_ADD_MANUAL: + case PARTITION_RETENTION: { p_buf = (uint8_t *)buf_get(&s->partition.u.tpt.period, sizeof(s->partition.u.tpt.period), p_buf, p_buf_end); p_buf = (uint8_t *)buf_get(&s->partition.u.tpt.retention, sizeof(s->partition.u.tpt.retention), p_buf, p_buf_end); diff --git a/schemachange/schemachange.h b/schemachange/schemachange.h index c7f750ac1e..c598c14581 100644 --- a/schemachange/schemachange.h +++ b/schemachange/schemachange.h @@ -66,6 +66,7 @@ enum comdb2_partition_type { PARTITION_NONE = 0, PARTITION_REMOVE = 1, PARTITION_MERGE = 2, + PARTITION_RETENTION = 3, /* change retention of an existing truncate partition */ PARTITION_ADD_TIMED = 20, PARTITION_ADD_MANUAL = 21, PARTITION_ADD_COL_RANGE = 40, diff --git a/sqlite/src/comdb2build.c b/sqlite/src/comdb2build.c index d7070c8b3d..f439905fc4 100644 --- a/sqlite/src/comdb2build.c +++ b/sqlite/src/comdb2build.c @@ -2984,7 +2984,9 @@ void comdb2timepartRetention(Parse *pParse, Token *nm, Token *lnm, int retention setError(pParse, SQLITE_ERROR, "Partition does not exist"); goto clean_arg; } else if (rc == ROLLOUT_TRUNC) { - setError(pParse, SQLITE_ERROR, "Use alter to change partition config"); + setError(pParse, SQLITE_ERROR, + "Use 'ALTER TABLE PARTITIONED BY TIME|MANUAL RETENTION " + "' to change a truncate partition's retention"); goto clean_arg; } @@ -5185,6 +5187,9 @@ void comdb2AlterTableEnd(Parse *pParse) } } else if (sc->partition.type == PARTITION_MERGE) { sc->force_rebuild = 1; + } else if (sc->partition.type == PARTITION_RETENTION) { + /* retention-only change: no column delta, no data movement, the + shards are added/dropped by the apply path itself. */ } } @@ -7948,6 +7953,16 @@ void comdb2CreateManualPartition(Parse *pParse, Token *retention, Token *start) { struct comdb2_partition *partition; + /* "PARTITIONED BY MANUAL RETENTION " (no START) against an existing + partition is an ALTER that changes the retention, not a create. */ + if (!start) { + struct comdb2_ddl_context *ctx = pParse->comdb2_ddl_ctx; + if (ctx && ctx->partition_first_shardname) { + comdb2AlterRetention(pParse, retention, 1 /* is_manual */); + return; + } + } + if (!gbl_partitioned_table_enabled) { setError(pParse, SQLITE_ABORT, "Create manual partitioned table not enabled"); return; @@ -7969,6 +7984,79 @@ void comdb2CreateManualPartition(Parse *pParse, Token *retention, Token *start) partition->u.tpt.start = tmp; } +/** + * Change the retention of an existing truncate time/manual partition: + * ALTER TABLE PARTITIONED BY TIME|MANUAL RETENTION + * Only the shard count changes; the period and data layout are untouched. + */ +void comdb2AlterRetention(Parse *pParse, Token *retention, int is_manual) +{ + struct comdb2_partition *partition; + int32_t ret = 0; + + if (comdb2IsPrepareOnly(pParse)) + return; + + if (!gbl_partitioned_table_enabled) { + setError(pParse, SQLITE_ABORT, "Partitioned table not enabled"); + return; + } + + /* remove=1: allowed to operate on an already-existing partition */ + partition = _get_partition(pParse, 1); + if (!partition) + return; + + if (_get_retention(retention, &ret)) { + setError(pParse, SQLITE_MISUSE, "Invalid retention"); + free_ddl_context(pParse); + return; + } + if (ret < 2) { + setError(pParse, SQLITE_ERROR, "Retention must be 2 or higher"); + free_ddl_context(pParse); + return; + } + + /* validate the target here so the client gets a clear message (apply-time + * schema-change errors are reported to the client only generically) */ + { + struct comdb2_ddl_context *ctx = pParse->comdb2_ddl_ctx; + const char *part = ctx ? ctx->tablename : ""; + int rollout = timepart_rollout(part); + int period; + + if (rollout == ROLLOUT_INVALID) { + setError(pParse, SQLITE_ERROR, "Partition does not exist"); + free_ddl_context(pParse); + return; + } + if (rollout != ROLLOUT_TRUNC) { + setError(pParse, SQLITE_ERROR, + "Legacy partition, use PUT TIME PARTITION ... RETENTION"); + free_ddl_context(pParse); + return; + } + period = timepart_get_period(part); + if ((period == VIEW_PARTITION_MANUAL) != (is_manual != 0)) { + setError(pParse, SQLITE_ERROR, + is_manual + ? "Partition is time based, use PARTITIONED BY TIME" + : "Partition is manual, use PARTITIONED BY MANUAL"); + free_ddl_context(pParse); + return; + } + } + + partition->type = PARTITION_RETENTION; + partition->u.tpt.retention = ret; + /* Stash the keyword used (TIME vs MANUAL) so the apply path can verify it + matches the partition kind; period is otherwise unused here. */ + partition->u.tpt.period = + is_manual ? VIEW_PARTITION_MANUAL : VIEW_PARTITION_INVALID; + partition->u.tpt.start = 0; +} + int comdb2VerifyGenShardKeyExists(Parse *pParse, IdList *cols) { struct comdb2_ddl_context *ctx = pParse->comdb2_ddl_ctx; diff --git a/sqlite/src/comdb2build.h b/sqlite/src/comdb2build.h index 7e818e52b0..a4220c60c4 100644 --- a/sqlite/src/comdb2build.h +++ b/sqlite/src/comdb2build.h @@ -127,6 +127,7 @@ void comdb2DropPartition(Parse* p, Token* name); void comdb2CreateTimePartition(Parse* p, Token* period, Token* retention, Token* start, int retro); void comdb2CreateManualPartition(Parse* p, Token* retention, Token* start); +void comdb2AlterRetention(Parse* p, Token* retention, int is_manual); void comdb2CreateGenShard(Parse* p, IdList *, IdList *); void comdb2SaveMergeTable(Parse* p, Token* name, Token* database, int alter); diff --git a/sqlite/src/parse.y b/sqlite/src/parse.y index 6f3a395dd7..65ed571034 100644 --- a/sqlite/src/parse.y +++ b/sqlite/src/parse.y @@ -251,8 +251,13 @@ partition_options ::= MANUAL RETENTION INTEGER(R) START INTEGER(S). { comdb2CreateManualPartition(pParse, &R, &S); } partition_options ::= MANUAL RETENTION INTEGER(R). { + /* On an existing partition this is an ALTER ... RETENTION change; on a new + table it creates a manual partition (handled inside the callee). */ comdb2CreateManualPartition(pParse, &R, 0); } +partition_options ::= TIME RETENTION INTEGER(R). { + comdb2AlterRetention(pParse, &R, 0); +} partition_options ::= COLUMNS LP idlist(X) RP ON LP idlist(Y) RP. { comdb2CreateGenShard(pParse, X, Y); } diff --git a/tests/timepart_retention_alter.test/Makefile b/tests/timepart_retention_alter.test/Makefile new file mode 100644 index 0000000000..b4c0ac1057 --- /dev/null +++ b/tests/timepart_retention_alter.test/Makefile @@ -0,0 +1,8 @@ +ifeq ($(TESTSROOTDIR),) + include ../testcase.mk +else + include $(TESTSROOTDIR)/testcase.mk +endif +ifeq ($(TEST_TIMEOUT),) + export TEST_TIMEOUT=5m +endif diff --git a/tests/timepart_retention_alter.test/README b/tests/timepart_retention_alter.test/README new file mode 100644 index 0000000000..59d33b756d --- /dev/null +++ b/tests/timepart_retention_alter.test/README @@ -0,0 +1,14 @@ +Change the retention of a truncate-rollout time partition via + ALTER TABLE PARTITIONED BY TIME RETENTION + +The change is applied inline/atomically: increasing retention creates new empty +shards (inserted so they are filled next, preserving existing data); decreasing +retention drops the oldest shards. Uses a manual (logical-cron) partition so +rollouts can be forced deterministically with `put counter`. + +Also checks that the legacy `PUT TIME PARTITION ... RETENTION` path still works for +legacy add&drop partitions, and that the two surfaces reject each other's partition +type. + +This test is self-validating (asserts and exits non-zero on failure); it does not +diff against a golden file. diff --git a/tests/timepart_retention_alter.test/lrl.options b/tests/timepart_retention_alter.test/lrl.options new file mode 100644 index 0000000000..7122a53f79 --- /dev/null +++ b/tests/timepart_retention_alter.test/lrl.options @@ -0,0 +1 @@ +legacy_tpt_partition 1 diff --git a/tests/timepart_retention_alter.test/runit b/tests/timepart_retention_alter.test/runit new file mode 100755 index 0000000000..d40b4a0c29 --- /dev/null +++ b/tests/timepart_retention_alter.test/runit @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +bash -n "$0" | exit 1 +source ${TESTSROOTDIR}/tools/runit_common.sh + +# Test: change retention of a truncate time partition via +# ALTER TABLE PARTITIONED BY TIME|MANUAL RETENTION +# applied inline (increase creates empty shards consumed-next; decrease drops +# the oldest shards). Also checks legacy PUT retention still works and that the +# two surfaces reject each other's partition type. + +dbnm=$1 +output=run.log +[ "x$dbnm" = "x" ] && { echo "need a DB name"; exit 1; } + +rm -f $output 2>/dev/null +touch $output + +CDB2="cdb2sql ${CDB2_OPTIONS} $dbnm default" + +fail() { echo "FAILURE: $1"; echo "FAILURE: $1" >> $output; dump_state; exit 1; } + +# run a statement, log it, require success +gorun() { + echo "> $1" >> $output + $CDB2 "$1" >> $output 2>&1 || fail "unexpected error running: $1" +} + +# scalar query value (tabs, trimmed) +sqlval() { + cdb2sql -tabs ${CDB2_OPTIONS} $dbnm default "$1" 2>/dev/null | tr -d '[:space:]' +} + +dump_state() { + echo "---- state dump ----" >> $output + cdb2sql -tabs ${CDB2_OPTIONS} $dbnm default \ + "select name, period, retention, nshards from comdb2_timepartitions" >> $output 2>&1 + cdb2sql -tabs ${CDB2_OPTIONS} $dbnm default \ + "select name, shardname from comdb2_timepartshards order by name, shardname" >> $output 2>&1 +} + +assert_shards() { # partition, expected count + local got=$(sqlval "select count(*) from comdb2_timepartshards where name='$1'") + echo "assert_shards($1): got=$got want=$2" >> $output + [ "$got" = "$2" ] || fail "partition '$1' shard count $got != $2" +} + +assert_retention() { # partition, expected retention (comdb2_timepartitions) + local got=$(sqlval "select retention from comdb2_timepartitions where name='$1'") + echo "assert_retention($1): got=$got want=$2" >> $output + [ "$got" = "$2" ] || fail "partition '$1' retention $got != $2" +} + +assert_total() { # partition(table), expected row count + local got=$(sqlval "select count(*) from $1") + echo "assert_total($1): got=$got want=$2" >> $output + [ "$got" = "$2" ] || fail "table '$1' row count $got != $2" +} + +assert_present() { # table, value of column a + local got=$(sqlval "select count(*) from $1 where a=$2") + echo "assert_present($1,$2): got=$got" >> $output + [ "$got" = "1" ] || fail "table '$1' expected row a=$2 present" +} + +assert_absent() { # table, predicate + local got=$(sqlval "select count(*) from $1 where $2") + echo "assert_absent($1,[$2]): got=$got" >> $output + [ "$got" = "0" ] || fail "table '$1' expected no rows where $2 (got $got)" +} + +# query -> newline separated values, whitespace-trimmed (unlike sqlval, does +# not collapse multiple rows into one) +sqllist() { + cdb2sql -tabs ${CDB2_OPTIONS} $dbnm default "$1" 2>/dev/null | sed 's/[[:space:]]*$//' +} + +# value, shard-list (newline separated) -> count of that value across those +# physical shard tables (shard names need single-quoting, they start with '$') +count_in_shards() { + local val=$1 shards=$2 total=0 s c + while IFS= read -r s; do + [ -z "$s" ] && continue + c=$(sqlval "select count(*) from '$s' where a=$val") + total=$((total + c)) + done <<< "$shards" + echo $total +} + +# run a statement expected to FAIL; optionally require message pattern +expect_fail() { # query, pattern + local err + echo "> (expect fail) $1" >> $output + err=$($CDB2 "$1" 2>&1) + if [ $? -eq 0 ]; then fail "expected failure but succeeded: $1"; fi + echo "$err" >> $output + if [ -n "$2" ] && ! grep -qiE "$2" <<< "$err"; then + fail "error for '$1' did not match '/$2/': $err" + fi +} + +# force a rollout on a manual partition and let the async truncate settle +roll() { + gorun "put counter $1 increment" + sleep 10 +} + +################################################################################ +echo "=== Section A: inline retention change, shard count + data preservation ===" >> $output + +gorun "create table t(a int) partitioned by manual retention 3 start 1" +assert_shards t 3 +assert_retention t 3 + +gorun "insert into t values(100)" +gorun "insert into t values(101)" +assert_total t 2 + +echo "--- increase 3 -> 5 (immediate) ---" >> $output +gorun "alter table t partitioned by manual retention 5" +assert_shards t 5 +assert_retention t 5 +assert_total t 2 # existing data preserved + +echo "--- decrease 5 -> 2 (immediate, empty shards dropped) ---" >> $output +gorun "alter table t partitioned by manual retention 2" +assert_shards t 2 +assert_retention t 2 +assert_total t 2 # data still there (dropped shards were empty) + +################################################################################ +echo "=== Section B: multiple rows/shard, rollouts after resize: increase keeps all data and uses the new shards, decrease drops the oldest data ===" >> $output + +gorun "create table tb(a int) partitioned by manual retention 2 start 1" +assert_shards tb 2 + +# seed each of the 2 original shards directly (by physical table name) with +# several rows, rather than relying on rollout timing via the partition alias +orig_shards=$(sqllist "select shardname from comdb2_timepartshards where name='tb' order by shardname") +old_values="" +i=0 +while IFS= read -r s; do + [ -z "$s" ] && continue + i=$((i+1)) + for v in $((i*10+1)) $((i*10+2)) $((i*10+3)); do + gorun "insert into '$s' values($v)" + old_values="$old_values $v" + done +done <<< "$orig_shards" +assert_total tb 6 + +echo "--- grow to 4 BEFORE any rollout: all existing (per-shard) data must survive ---" >> $output +gorun "alter table tb partitioned by manual retention 4" +assert_shards tb 4 +assert_total tb 6 + +all_shards=$(sqllist "select shardname from comdb2_timepartshards where name='tb' order by shardname") +new_shards=$(comm -13 <(sort <<< "$orig_shards") <(sort <<< "$all_shards")) +echo "new shards after growth: $new_shards" >> $output +[ "$(wc -l <<< "$new_shards")" = "2" ] || fail "expected 2 new shards after growing retention, got: $new_shards" + +echo "--- 2 manual rollouts: each insert (via the partition alias) must land on a newly created shard ---" >> $output +roll tb +gorun "insert into tb values(101)" +c=$(count_in_shards 101 "$new_shards") +echo "row 101 (post rollout #1) found in $c new shard(s)" >> $output +[ "$c" = "1" ] || fail "row 101 was not written to one of the newly created shards" + +roll tb +gorun "insert into tb values(102)" +c=$(count_in_shards 102 "$new_shards") +echo "row 102 (post rollout #2) found in $c new shard(s)" >> $output +[ "$c" = "1" ] || fail "row 102 was not written to one of the newly created shards" + +echo "--- no data lost across growth + 2 rollouts ---" >> $output +assert_total tb 8 +for v in $old_values 101 102; do assert_present tb $v; done + +echo "--- shrink to 2: the two OLDEST shards (all pre-growth, per-shard-seeded data) are dropped ---" >> $output +# snapshot every shard's [low,high] before the shrink; decrease must drop the +# oldest shards but leave every surviving shard's limits untouched +bounds_before=$(sqllist "select shardname||' '||low||' '||high from comdb2_timepartshards where name='tb'") +gorun "alter table tb partitioned by manual retention 2" +assert_shards tb 2 +assert_absent tb "a in ($(echo $old_values | tr ' ' ','))" +assert_present tb 101 +assert_present tb 102 +assert_total tb 2 + +while read -r sn lo hi; do + [ -z "$sn" ] && continue + after=$(sqlval "select low||'/'||high from comdb2_timepartshards where name='tb' and shardname='$sn'") + [ -z "$after" ] && continue # this shard was one of the dropped oldest ones + echo "surviving shard $sn [low/high] across decrease: $lo/$hi -> $after" >> $output + [ "$after" = "$lo/$hi" ] || fail "decrease altered surviving shard $sn limits: $lo/$hi -> $after" +done <<< "$bounds_before" + +################################################################################ +echo "=== Section C: validation / error cases ===" >> $output + +# retention below the minimum +expect_fail "alter table t partitioned by manual retention 1" "2 or higher" +# wrong keyword for the partition kind (t is manual) +expect_fail "alter table t partitioned by time retention 4" "manual|MANUAL" +# truncate partition cannot use the legacy PUT surface +expect_fail "put time partition t retention 4" "alter" + +################################################################################ +echo "=== Section D: legacy add&drop partition still uses PUT ===" >> $output + +gorun "create table lt(a int)" +starttime=$(get_timestamp 3600) # get_timestamp already appends the timezone +gorun "CREATE TIME PARTITION ON lt as ltp PERIOD 'daily' RETENTION 2 START '$starttime'" +assert_retention ltp 2 + +echo "--- legacy PUT retention still works ---" >> $output +gorun "put time partition ltp retention 3" +assert_retention ltp 3 + +echo "--- ALTER retention rejected on a legacy partition ---" >> $output +expect_fail "alter table ltp partitioned by time retention 4" "legacy|put|PUT" + +################################################################################ +dump_state +echo "SUCCESS" >> $output +echo "SUCCESS" diff --git a/tests/timepart_retention_alter.test/t.csc2 b/tests/timepart_retention_alter.test/t.csc2 new file mode 100644 index 0000000000..fa5166b18f --- /dev/null +++ b/tests/timepart_retention_alter.test/t.csc2 @@ -0,0 +1,8 @@ +schema +{ + int a +} + +keys { + "PK"=a +}