From 601c7fd11ba266e135ff42ee5098669e2d79a095 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 6 Jul 2026 15:19:47 -0300 Subject: [PATCH 01/16] feat: Add opt-in SqrlPagination metadata for paginated GraphQL queries Signed-off-by: Marvin Froeder --- .../datasqrl/config/GraphqlSourceLoader.java | 15 +- .../server/GraphqlModelGenerator.java | 27 +- .../server/GraphqlSchemaValidator.java | 17 +- .../datasqrl/server/GraphqlSchemaWalker.java | 28 +- .../datasqrl/server/SqrlPaginationUtil.java | 200 ++ .../server/graphql/RootGraphQLModel.java | 17 +- .../datasqrl/server/jdbc/VertxJdbcClient.java | 8 +- .../jdbc/VertxQueryExecutionContext.java | 104 +- .../java/com/datasqrl/server/WriteIT.java | 3 +- .../server/jdbc/PaginationMetadataTest.java | 98 + ...veTest-fail-paged-no-limit-offset.graphqls | 21 + ...nsiveTest-fail-paged-subscription.graphqls | 25 + ...siveTest-fail-paged-unknown-field.graphqls | 22 + ...-fail-paged-wrong-pagination-type.graphqls | 26 + .../comprehensiveTest-paged-results.graphqls | 21 + ...mprehensiveTest-paged-userdefined.graphqls | 40 + ...hensiveTest-fail-paged-no-limit-offset.txt | 28 + ...prehensiveTest-fail-paged-subscription.txt | 28 + ...rehensiveTest-fail-paged-unknown-field.txt | 28 + ...eTest-fail-paged-wrong-pagination-type.txt | 41 + ...ehensiveTest-limit-offset-combinations.txt | 167 ++ .../comprehensiveTest-paged-results.txt | 1819 ++++++++++++++++ .../comprehensiveTest-paged-userdefined.txt | 1842 +++++++++++++++++ .../comprehensiveTest-parameters-order.txt | 167 ++ .../comprehensiveTest.txt | 167 ++ 25 files changed, 4946 insertions(+), 13 deletions(-) create mode 100644 sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java create mode 100644 sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-subscription.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-unknown-field.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java index 85a67c8dd2..ab4e53a672 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java @@ -24,6 +24,7 @@ import com.datasqrl.server.ApiSources; import com.datasqrl.server.GraphqlSchemaHandler; import com.datasqrl.server.ScriptFiles; +import com.datasqrl.server.SqrlPaginationUtil; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -75,9 +76,17 @@ public LoadResult load(ServerPhysicalPlan serverPlan) { } if (!shouldUseInferredSchema(apiVersions)) { - apiVersions.forEach( - apiVersion -> graphqlSchemaHandler.validateSchema(apiVersion, serverPlan)); - return new LoadResult(apiVersions, Optional.empty()); + var injected = + apiVersions.stream() + .map( + apiVersion -> + new ApiSources( + apiVersion.version(), + SqrlPaginationUtil.injectPaginationType(apiVersion.schema()), + apiVersion.operations())) + .toList(); + injected.forEach(apiVersion -> graphqlSchemaHandler.validateSchema(apiVersion, serverPlan)); + return new LoadResult(injected, Optional.empty()); } List operations; diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index 9c6e42e9bc..777f291f7b 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -148,7 +148,8 @@ protected void visitQuery( ObjectTypeDefinition parentType, FieldDefinition atField, SqrlTableFunction tableFunction, - TypeDefinitionRegistry registry) { + TypeDefinitionRegistry registry, + boolean paged) { // As we no more merge user provided graphQL schema with the inferred schema, we no more need to // generate as many queries as the permutations of its arguments. // We now have a single executable query linked to the table function and already fully defined @@ -168,13 +169,15 @@ protected void visitQuery( .map(InputValueDefinition::getName) .anyMatch( name -> name.equals(SchemaConstants.LIMIT) || name.equals(SchemaConstants.OFFSET)); + var countSql = paged ? buildCountSql(tableFunction, executableJdbcReadQuery.getSql()) : null; queryBase = new SqlQuery( executableJdbcReadQuery.getSql(), parameters, hasLimitOrOffset ? PaginationType.LIMIT_AND_OFFSET : PaginationType.NONE, executableJdbcReadQuery.getCacheDuration().toMillis(), - executableJdbcReadQuery.getDatabase()); + executableJdbcReadQuery.getDatabase(), + countSql); var coordsBuilder = ArgumentLookupQueryCoords.builder() .parentType(parentType.getName()) @@ -186,6 +189,26 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } + /** + * Builds the companion COUNT(*) query for a paginated result. Adds MIN/MAX over the designated + * rowtime column when the result has one, so {@code firstEventTime}/{@code lastEventTime} can be + * populated. The rowtime column name is the same identifier as in the base query's output. + */ + private static String buildCountSql(SqrlTableFunction tableFunction, String baseSql) { + var tsCol = + tableFunction.getRowTime().map(tableFunction::getField).map(RelDataTypeField::getName); + var select = new StringBuilder("SELECT COUNT(*) AS \"total_records\""); + tsCol.ifPresent( + col -> + select + .append(", MIN(\"") + .append(col) + .append("\") AS \"first_event_time\", MAX(\"") + .append(col) + .append("\") AS \"last_event_time\"")); + return select.append(" FROM (").append(baseSql).append(") x").toString(); + } + private static QueryParameterHandler convert(FunctionParameter fnParam) { final var sqrlParam = (SqrlFunctionParameter) fnParam; diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java index 9bb4026ee4..4d28afb5f9 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java @@ -374,8 +374,23 @@ protected void visitQuery( ObjectTypeDefinition parentType, FieldDefinition atField, SqrlTableFunction tableFunction, - TypeDefinitionRegistry registry) { + TypeDefinitionRegistry registry, + boolean paged) { checkValidArrayNonNullType(atField.getType()); + if (paged) { + SqrlPaginationUtil.validatePaginationType(registry); + var argNames = + atField.getInputValueDefinitions().stream() + .map(InputValueDefinition::getName) + .collect(Collectors.toSet()); + checkState( + argNames.contains(LIMIT) && argNames.contains(OFFSET), + atField.getSourceLocation(), + "Paginated query [%s] must declare both '%s' and '%s' arguments", + atField.getName(), + LIMIT, + OFFSET); + } checkArgumentsMatchParameters(atField, tableFunction, registry); } diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java index 6900f5ee91..632a086f80 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java @@ -23,6 +23,7 @@ import com.datasqrl.canonicalizer.Name; import com.datasqrl.canonicalizer.NamePath; +import com.datasqrl.plan.table.Multiplicity; import com.datasqrl.planner.dag.plan.MutationTable; import com.datasqrl.planner.parser.AccessModifier; import com.datasqrl.planner.tables.SqrlTableFunction; @@ -138,14 +139,34 @@ private void walkTableFunction( typeDefinition.getSourceLocation(), "Could not infer non-object type on graphql schema: %s", typeDefinition.getName()); + var resultType = (ObjectTypeDefinition) typeDefinition; + + // A page wrapper ({results: [Element!], pagination: SqrlPagination}) is treated like a list of + // Element: validate/walk the element type against the function row type and compute pagination + // metadata via a companion count query. + var pagedElement = SqrlPaginationUtil.getPagedElementType(resultType, registry); + var paged = pagedElement.isPresent(); + if (paged) { + checkState( + tableFunction.getVisibility().access() == AccessModifier.QUERY, + atField.getSourceLocation(), + "Paginated result types are only supported for queries: %s", + atField.getName()); + checkState( + tableFunction.getMultiplicity() == Multiplicity.MANY, + atField.getSourceLocation(), + "Paginated result types require a multi-row result (no LIMIT 1): %s", + atField.getName()); + resultType = pagedElement.get(); + } + if (tableFunction.getVisibility().access() == AccessModifier.QUERY) { // walking a query table function - visitQuery(parentType, atField, tableFunction, registry); + visitQuery(parentType, atField, tableFunction, registry, paged); } else { // walking a subscription table function visitSubscription(atField, tableFunction, registry); } var functionRowType = tableFunction.getRowType(); - var resultType = (ObjectTypeDefinition) typeDefinition; walkObjectType(true, resultType, Optional.of(functionRowType), registry); } @@ -257,7 +278,8 @@ protected abstract void visitQuery( ObjectTypeDefinition parentType, FieldDefinition atField, SqrlTableFunction tableFunction, - TypeDefinitionRegistry registry); + TypeDefinitionRegistry registry, + boolean paged); protected abstract void visitSubscription( FieldDefinition atField, SqrlTableFunction tableFunction, TypeDefinitionRegistry registry); diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java new file mode 100644 index 0000000000..a88c5ab994 --- /dev/null +++ b/sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java @@ -0,0 +1,200 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.server; + +import static com.datasqrl.server.util.GraphqlCheckUtil.checkState; + +import graphql.language.FieldDefinition; +import graphql.language.ListType; +import graphql.language.NonNullType; +import graphql.language.ObjectTypeDefinition; +import graphql.language.Type; +import graphql.language.TypeName; +import graphql.schema.idl.SchemaParser; +import graphql.schema.idl.TypeDefinitionRegistry; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Opt-in pagination support: a query whose result type is a page wrapper ({@code {results: + * [Element!] pagination: SqrlPagination}}) returns its rows plus pagination metadata computed from + * a companion COUNT(*) query. This util detects the wrapper shape and injects/validates the + * standard {@code SqrlPagination} type. + */ +public final class SqrlPaginationUtil { + + private SqrlPaginationUtil() {} + + public static final String PAGINATION_TYPE_NAME = "SqrlPagination"; + + /** Canonical field -> printed type, kept in declaration order for the injected SDL. */ + private static final Map PAGINATION_FIELDS = new LinkedHashMap<>(); + + static { + PAGINATION_FIELDS.put("totalRecords", "Long!"); + PAGINATION_FIELDS.put("pageSize", "Int!"); + PAGINATION_FIELDS.put("currentPage", "Int!"); + PAGINATION_FIELDS.put("totalPages", "Int!"); + PAGINATION_FIELDS.put("hasNextPage", "Boolean!"); + PAGINATION_FIELDS.put("hasPreviousPage", "Boolean!"); + PAGINATION_FIELDS.put("nextOffset", "Int"); + PAGINATION_FIELDS.put("prevOffset", "Int"); + PAGINATION_FIELDS.put("firstEventTime", "DateTime"); + PAGINATION_FIELDS.put("lastEventTime", "DateTime"); + } + + private static final String CANONICAL_SDL = buildCanonicalSdl(); + + private static String buildCanonicalSdl() { + var sb = new StringBuilder("type ").append(PAGINATION_TYPE_NAME).append(" {\n"); + PAGINATION_FIELDS.forEach( + (name, type) -> sb.append(" ").append(name).append(": ").append(type).append("\n")); + return sb.append("}\n").toString(); + } + + /** + * If the schema references {@code SqrlPagination} but does not define it, append the canonical + * definition (plus any missing scalar declarations). A user-provided definition is left untouched + * here and validated later by {@link #validatePaginationType} within the schema validator's error + * scope. Returns the (possibly rewritten) source. + */ + public static ApiSource injectPaginationType(ApiSource schema) { + TypeDefinitionRegistry registry; + try { + registry = new SchemaParser().parse(schema.getDefinition()); + } catch (Exception e) { + // Let the downstream validator report parse errors with proper source location; an + // unparseable schema cannot reference the pagination type anyway. + return schema; + } + + if (!referencesPaginationType(registry) || registry.getType(PAGINATION_TYPE_NAME).isPresent()) { + return schema; + } + + var injected = new StringBuilder(schema.getDefinition()); + injected.append("\n"); + if (registry.scalars().get("Long") == null) { + injected.append("scalar Long\n"); + } + if (registry.scalars().get("DateTime") == null) { + injected.append("scalar DateTime\n"); + } + injected.append(CANONICAL_SDL); + + return new ApiSource(schema.getPath().orElse(null), injected.toString()); + } + + /** + * Validates that a user-provided {@code SqrlPagination} type matches the canonical definition. + * Must be called from within the schema validator so mismatches are reported like other schema + * errors. No-op when the type is absent (it will have been injected) or unreferenced. + */ + public static void validatePaginationType(TypeDefinitionRegistry registry) { + var existing = registry.getType(PAGINATION_TYPE_NAME); + if (existing.isEmpty()) { + return; + } + checkState( + existing.get() instanceof ObjectTypeDefinition, + existing.get().getSourceLocation(), + "%s must be an object type", + PAGINATION_TYPE_NAME); + validateMatchesCanonical((ObjectTypeDefinition) existing.get()); + } + + /** + * Detects the page wrapper shape: an object type with exactly two fields, one a list of an object + * type (the results) and the other of type {@code SqrlPagination}. Returns the element object + * type when the shape matches. + */ + public static Optional getPagedElementType( + ObjectTypeDefinition wrapper, TypeDefinitionRegistry registry) { + var fields = wrapper.getFieldDefinitions(); + if (fields.size() != 2) { + return Optional.empty(); + } + + ObjectTypeDefinition elementType = null; + boolean hasPagination = false; + for (FieldDefinition field : fields) { + var type = unwrapNonNull(field.getType()); + if (type instanceof ListType listType) { + var element = unwrapNonNull(listType.getType()); + if (element instanceof TypeName typeName) { + elementType = + registry + .getType(typeName.getName()) + .filter(t -> t instanceof ObjectTypeDefinition) + .map(t -> (ObjectTypeDefinition) t) + .orElse(null); + } + } else if (type instanceof TypeName typeName + && PAGINATION_TYPE_NAME.equals(typeName.getName())) { + hasPagination = true; + } + } + + return hasPagination ? Optional.ofNullable(elementType) : Optional.empty(); + } + + private static boolean referencesPaginationType(TypeDefinitionRegistry registry) { + return registry.types().values().stream() + .filter(t -> t instanceof ObjectTypeDefinition) + .flatMap(t -> ((ObjectTypeDefinition) t).getFieldDefinitions().stream()) + .anyMatch(field -> referencesPaginationType(field.getType())); + } + + private static boolean referencesPaginationType(Type type) { + var unwrapped = unwrapNonNull(type); + if (unwrapped instanceof ListType listType) { + return referencesPaginationType(listType.getType()); + } + return unwrapped instanceof TypeName typeName + && PAGINATION_TYPE_NAME.equals(typeName.getName()); + } + + private static void validateMatchesCanonical(ObjectTypeDefinition userType) { + var actual = new LinkedHashMap(); + for (FieldDefinition field : userType.getFieldDefinitions()) { + actual.put(field.getName(), printType(field.getType())); + } + checkState( + actual.equals(PAGINATION_FIELDS), + userType.getSourceLocation(), + "User-defined %s does not match the expected definition:\n%s", + PAGINATION_TYPE_NAME, + CANONICAL_SDL); + } + + private static String printType(Type type) { + if (type instanceof NonNullType nonNull) { + return printType(nonNull.getType()) + "!"; + } + if (type instanceof ListType listType) { + return "[" + printType(listType.getType()) + "]"; + } + if (type instanceof TypeName typeName) { + return typeName.getName(); + } + return type.toString(); + } + + private static Type unwrapNonNull(Type type) { + return type instanceof NonNullType nonNull ? unwrapNonNull(nonNull.getType()) : type; + } +} diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java index b78cd11c8c..d1fcf58d5e 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java @@ -20,6 +20,7 @@ import com.datasqrl.server.jdbc.DatabaseType; import com.datasqrl.server.operation.ApiOperation; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -298,13 +299,20 @@ public static class SqlQuery implements QueryBase { /** The database the query is executed against */ DatabaseType database; + /** + * Companion COUNT(*) query producing pagination metadata. When non-null, the query returns a + * page wrapper ({@code {results, pagination}}) rather than a bare list. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + String countSql; + @Override public R accept(QueryBaseVisitor visitor, C context) { return visitor.visitSqlQuery(this, context); } public SqlQuery updateSql(String newSql) { - return new SqlQuery(newSql, parameters, pagination, cacheDurationMs, database); + return new SqlQuery(newSql, parameters, pagination, cacheDurationMs, database, countSql); } } @@ -487,6 +495,13 @@ public static class ResolvedSqlQuery implements ResolvedQuery { SqlQuery query; PreparedSqrlQuery preparedQueryContainer; + /** Prepared companion count query; null for non-paged queries or non-binding databases. */ + PreparedSqrlQuery preparedCountQueryContainer; + + public ResolvedSqlQuery(SqlQuery query, PreparedSqrlQuery preparedQueryContainer) { + this(query, preparedQueryContainer, null); + } + @Override public R accept(ResolvedQueryVisitor visitor, C context) { return visitor.visitResolvedSqlQuery(this, context); diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java index 03dd567292..26c7dc942f 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java @@ -44,7 +44,13 @@ public ResolvedQuery prepareQuery(SqlQuery query, ServerContext context) { var preparedQuery = sqlClient.preparedQuery(query.getSql()); - return new ResolvedSqlQuery(query, new PreparedVertxSqrlQuery(preparedQuery)); + PreparedVertxSqrlQuery preparedCountQuery = null; + if (query.getCountSql() != null) { + preparedCountQuery = new PreparedVertxSqrlQuery(sqlClient.preparedQuery(query.getCountSql())); + } + + return new ResolvedSqlQuery( + query, new PreparedVertxSqrlQuery(preparedQuery), preparedCountQuery); } @Override diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index bbf702cf01..092434ccad 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -23,7 +23,12 @@ import com.datasqrl.server.graphql.RootGraphQLModel.Argument; import com.datasqrl.server.graphql.RootGraphQLModel.ResolvedSqlQuery; import graphql.schema.DataFetchingEnvironment; +import graphql.schema.GraphQLList; +import graphql.schema.GraphQLNonNull; +import graphql.schema.GraphQLObjectType; +import graphql.schema.GraphQLOutputType; import io.vertx.core.Future; +import io.vertx.core.json.JsonObject; import io.vertx.sqlclient.Row; import io.vertx.sqlclient.RowSet; import io.vertx.sqlclient.Tuple; @@ -72,14 +77,25 @@ private CompletableFuture runQueryInternal( ResolvedSqlQuery resolvedQuery, boolean isList, List paramObj) { var preparedQueryContainer = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedQueryContainer(); + var countContainer = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedCountQueryContainer(); var query = resolvedQuery.getQuery(); var unpreparedSqlQuery = query.getSql(); + var database = query.getDatabase(); + var paged = query.getCountSql() != null; + + // The count query is bound with the base parameters only, without the runtime limit/offset. + var countParams = paged ? Tuple.from(List.copyOf(paramObj)) : null; + + int limitValue = Integer.MAX_VALUE; + int offsetValue = 0; switch (query.getPagination()) { case NONE: break; case LIMIT_AND_OFFSET: var limit = Optional.ofNullable(environment.getArgument(LIMIT)); var offset = Optional.ofNullable(environment.getArgument(OFFSET)); + limitValue = limit.orElse(Integer.MAX_VALUE); + offsetValue = offset.orElse(0); // special case where database doesn't support binding for limit/offset => need // to execute dynamically @@ -102,7 +118,6 @@ private CompletableFuture runQueryInternal( // execute the preparedQuery with the arguments extracted above Future> future; var params = Tuple.from(paramObj); - var database = resolvedQuery.getQuery().getDatabase(); if (preparedQueryContainer == null) { future = serverContext.getSqlClient().execute(database, unpreparedSqlQuery, params); @@ -111,6 +126,28 @@ private CompletableFuture runQueryInternal( future = serverContext.getSqlClient().execute(preparedQuery, params); } + if (paged) { + Future> countFuture = + countContainer == null + ? serverContext.getSqlClient().execute(database, query.getCountSql(), countParams) + : serverContext.getSqlClient().execute(countContainer.preparedQuery(), countParams); + var dataFuture = future; + var effectiveLimit = limitValue; + var effectiveOffset = offsetValue; + Future.all(dataFuture, countFuture) + .map( + c -> + pagedResultMapper( + dataFuture.result(), countFuture.result(), effectiveLimit, effectiveOffset)) + .onSuccess(cf::complete) + .onFailure( + f -> { + f.printStackTrace(); + cf.completeExceptionally(f); + }); + return cf; + } + // map the resultSet to json for GraphQL response future .map(r -> resultMapper(r, isList)) @@ -128,4 +165,69 @@ private Object resultMapper(RowSet r, boolean isList) { return unboxList(o, isList); } + + private Object pagedResultMapper( + RowSet dataRows, RowSet countRows, int limit, int offset) { + var results = StreamSupport.stream(dataRows.spliterator(), false).map(Row::toJson).toList(); + + var countIterator = countRows.iterator(); + var countJson = countIterator.hasNext() ? countIterator.next().toJson() : new JsonObject(); + var totalRecords = countJson.getLong("total_records", 0L); + var pagination = + buildPaginationMetadata( + totalRecords, + limit, + offset, + countJson.getValue("first_event_time"), + countJson.getValue("last_event_time")); + + var fieldNames = pageFieldNames(environment.getFieldType()); + return new JsonObject() + .put(fieldNames.resultsField(), results) + .put(fieldNames.paginationField(), pagination); + } + + /** Derives the results/pagination field names from the page wrapper's GraphQL object type. */ + private static PageFieldNames pageFieldNames(GraphQLOutputType fieldType) { + if (fieldType instanceof GraphQLNonNull g) { + fieldType = (GraphQLOutputType) g.getWrappedType(); + } + var objectType = (GraphQLObjectType) fieldType; + String resultsField = null; + String paginationField = null; + for (var field : objectType.getFieldDefinitions()) { + var type = field.getType(); + if (type instanceof GraphQLNonNull g) { + type = (GraphQLOutputType) g.getWrappedType(); + } + if (type instanceof GraphQLList) { + resultsField = field.getName(); + } else { + paginationField = field.getName(); + } + } + return new PageFieldNames(resultsField, paginationField); + } + + private record PageFieldNames(String resultsField, String paginationField) {} + + static JsonObject buildPaginationMetadata( + long totalRecords, int limit, int offset, Object firstEventTime, Object lastEventTime) { + int totalPages = limit == 0 ? 0 : (int) Math.ceil((double) totalRecords / limit); + int currentPage = limit == 0 ? 1 : offset / limit + 1; + boolean hasNextPage = (long) offset + limit < totalRecords; + boolean hasPreviousPage = offset > 0; + + return new JsonObject() + .put("totalRecords", totalRecords) + .put("pageSize", limit) + .put("currentPage", currentPage) + .put("totalPages", totalPages) + .put("hasNextPage", hasNextPage) + .put("hasPreviousPage", hasPreviousPage) + .put("nextOffset", hasNextPage ? Integer.valueOf(offset + limit) : null) + .put("prevOffset", hasPreviousPage ? Integer.valueOf(Math.max(0, offset - limit)) : null) + .put("firstEventTime", firstEventTime) + .put("lastEventTime", lastEventTime); + } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java index 4db031b61e..63d24ce7b1 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java @@ -171,7 +171,8 @@ private RootGraphQLModel getCustomerModel() { List.of(), PaginationType.NONE, 0, - DatabaseType.POSTGRES)) + DatabaseType.POSTGRES, + null)) .build()) .build()) .mutation( diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java new file mode 100644 index 0000000000..dec6f9ed46 --- /dev/null +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.server.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class PaginationMetadataTest { + + @Test + void givenEmptyResult_whenBuildMetadata_thenSinglePageNoRecords() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(0, 10, 0, null, null); + + assertThat(json.getLong("totalRecords")).isZero(); + assertThat(json.getInteger("pageSize")).isEqualTo(10); + assertThat(json.getInteger("currentPage")).isEqualTo(1); + assertThat(json.getInteger("totalPages")).isZero(); + assertThat(json.getBoolean("hasNextPage")).isFalse(); + assertThat(json.getBoolean("hasPreviousPage")).isFalse(); + assertThat(json.getInteger("nextOffset")).isNull(); + assertThat(json.getInteger("prevOffset")).isNull(); + } + + @Test + void givenFirstPage_whenBuildMetadata_thenHasNextNoPrevious() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 0, null, null); + + assertThat(json.getInteger("currentPage")).isEqualTo(1); + assertThat(json.getInteger("totalPages")).isEqualTo(3); + assertThat(json.getBoolean("hasNextPage")).isTrue(); + assertThat(json.getBoolean("hasPreviousPage")).isFalse(); + assertThat(json.getInteger("nextOffset")).isEqualTo(10); + assertThat(json.getInteger("prevOffset")).isNull(); + } + + @Test + void givenMiddlePage_whenBuildMetadata_thenHasBothNeighbours() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 10, null, null); + + assertThat(json.getInteger("currentPage")).isEqualTo(2); + assertThat(json.getBoolean("hasNextPage")).isTrue(); + assertThat(json.getBoolean("hasPreviousPage")).isTrue(); + assertThat(json.getInteger("nextOffset")).isEqualTo(20); + assertThat(json.getInteger("prevOffset")).isZero(); + } + + @Test + void givenLastPartialPage_whenBuildMetadata_thenNoNextHasPrevious() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 20, null, null); + + assertThat(json.getInteger("currentPage")).isEqualTo(3); + assertThat(json.getBoolean("hasNextPage")).isFalse(); + assertThat(json.getBoolean("hasPreviousPage")).isTrue(); + assertThat(json.getInteger("nextOffset")).isNull(); + assertThat(json.getInteger("prevOffset")).isEqualTo(10); + } + + @Test + void givenOffsetBeyondTotal_whenBuildMetadata_thenNoNextPage() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 30, null, null); + + assertThat(json.getBoolean("hasNextPage")).isFalse(); + assertThat(json.getBoolean("hasPreviousPage")).isTrue(); + assertThat(json.getInteger("prevOffset")).isEqualTo(20); + } + + @Test + void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 0, 0, null, null); + + assertThat(json.getInteger("totalPages")).isZero(); + assertThat(json.getInteger("currentPage")).isEqualTo(1); + } + + @Test + void givenEventTimes_whenBuildMetadata_thenPassedThrough() { + var json = + VertxQueryExecutionContext.buildPaginationMetadata( + 5, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); + + assertThat(json.getString("firstEventTime")).isEqualTo("2024-01-01T00:00:00Z"); + assertThat(json.getString("lastEventTime")).isEqualTo("2024-01-02T00:00:00Z"); + } +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls new file mode 100644 index 0000000000..678970c578 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls @@ -0,0 +1,21 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls new file mode 100644 index 0000000000..edf18ab299 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls @@ -0,0 +1,25 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls new file mode 100644 index 0000000000..3b7b998789 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls @@ -0,0 +1,22 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls new file mode 100644 index 0000000000..1dd177af24 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls @@ -0,0 +1,26 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls new file mode 100644 index 0000000000..9304f1b873 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls @@ -0,0 +1,21 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls new file mode 100644 index 0000000000..9809dd9dad --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls @@ -0,0 +1,40 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt new file mode 100644 index 0000000000..dbdfcf10cc --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt @@ -0,0 +1,28 @@ +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database._Customer[timestamp] +in script:comprehensiveTest.sqrl [10:1]: +CustomerFilteredDistinct := DISTINCT Customer ON customerid ORDER BY lastUpdated DESC; + +AnotherCustomer := SELECT customerid, email, lastUpdated FROM _Customer WHERE customerid > 100; +^ + +[NOTICE] You can rewrite the join as a temporal join for greater efficiency by adding: FOR SYSTEM_TIME AS OF `time` +in script:comprehensiveTest.sqrl [18:1]: +InvalidDistinct := SELECT customerid, `timestamp`, name AS namee FROM (SELECT *, (ROW_NUMBER() OVER (PARTITION BY customerid ORDER BY `timestamp` DESC)) AS _rownum FROM Customer) WHERE (_rownum = 1); + +MissedTemporalJoin := SELECT * FROM ExternalOrders o JOIN ExplicitDistinct c ON o.customerid = c.customerid; +^ + +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database.SelectCustomers[timestamp] +in script:comprehensiveTest.sqrl [51:1]: +); + +CustomerTimeWindow := SELECT +^ + +[FATAL] Paginated query [CustomerByTime2] must declare both 'limit' and 'offset' arguments +in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [20:5]: + +type Query { + CustomerByTime2: CustomerPage! +----^ + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-subscription.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-subscription.txt new file mode 100644 index 0000000000..a980021259 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-subscription.txt @@ -0,0 +1,28 @@ +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database._Customer[timestamp] +in script:comprehensiveTest.sqrl [10:1]: +CustomerFilteredDistinct := DISTINCT Customer ON customerid ORDER BY lastUpdated DESC; + +AnotherCustomer := SELECT customerid, email, lastUpdated FROM _Customer WHERE customerid > 100; +^ + +[NOTICE] You can rewrite the join as a temporal join for greater efficiency by adding: FOR SYSTEM_TIME AS OF `time` +in script:comprehensiveTest.sqrl [18:1]: +InvalidDistinct := SELECT customerid, `timestamp`, name AS namee FROM (SELECT *, (ROW_NUMBER() OVER (PARTITION BY customerid ORDER BY `timestamp` DESC)) AS _rownum FROM Customer) WHERE (_rownum = 1); + +MissedTemporalJoin := SELECT * FROM ExternalOrders o JOIN ExplicitDistinct c ON o.customerid = c.customerid; +^ + +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database.SelectCustomers[timestamp] +in script:comprehensiveTest.sqrl [51:1]: +); + +CustomerTimeWindow := SELECT +^ + +[FATAL] Paginated result types are only supported for queries: CustomerSubscription +in script:comprehensiveTest-fail-paged-subscription.graphqls [24:5]: + +type Subscription { + CustomerSubscription: CustomerPage +----^ + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-unknown-field.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-unknown-field.txt new file mode 100644 index 0000000000..dc5e91431a --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-unknown-field.txt @@ -0,0 +1,28 @@ +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database._Customer[timestamp] +in script:comprehensiveTest.sqrl [10:1]: +CustomerFilteredDistinct := DISTINCT Customer ON customerid ORDER BY lastUpdated DESC; + +AnotherCustomer := SELECT customerid, email, lastUpdated FROM _Customer WHERE customerid > 100; +^ + +[NOTICE] You can rewrite the join as a temporal join for greater efficiency by adding: FOR SYSTEM_TIME AS OF `time` +in script:comprehensiveTest.sqrl [18:1]: +InvalidDistinct := SELECT customerid, `timestamp`, name AS namee FROM (SELECT *, (ROW_NUMBER() OVER (PARTITION BY customerid ORDER BY `timestamp` DESC)) AS _rownum FROM Customer) WHERE (_rownum = 1); + +MissedTemporalJoin := SELECT * FROM ExternalOrders o JOIN ExplicitDistinct c ON o.customerid = c.customerid; +^ + +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database.SelectCustomers[timestamp] +in script:comprehensiveTest.sqrl [51:1]: +); + +CustomerTimeWindow := SELECT +^ + +[FATAL] Unknown field at location doesNotExist. Possible scalars are [customerid, email, name, lastUpdated, timestamp] +in script:comprehensiveTest-fail-paged-unknown-field.graphqls [12:5]: + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +----^ + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt new file mode 100644 index 0000000000..6908b873db --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt @@ -0,0 +1,41 @@ +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database._Customer[timestamp] +in script:comprehensiveTest.sqrl [10:1]: +CustomerFilteredDistinct := DISTINCT Customer ON customerid ORDER BY lastUpdated DESC; + +AnotherCustomer := SELECT customerid, email, lastUpdated FROM _Customer WHERE customerid > 100; +^ + +[NOTICE] You can rewrite the join as a temporal join for greater efficiency by adding: FOR SYSTEM_TIME AS OF `time` +in script:comprehensiveTest.sqrl [18:1]: +InvalidDistinct := SELECT customerid, `timestamp`, name AS namee FROM (SELECT *, (ROW_NUMBER() OVER (PARTITION BY customerid ORDER BY `timestamp` DESC)) AS _rownum FROM Customer) WHERE (_rownum = 1); + +MissedTemporalJoin := SELECT * FROM ExternalOrders o JOIN ExplicitDistinct c ON o.customerid = c.customerid; +^ + +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database.SelectCustomers[timestamp] +in script:comprehensiveTest.sqrl [51:1]: +); + +CustomerTimeWindow := SELECT +^ + +[FATAL] User-defined SqrlPagination does not match the expected definition: +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +in script:comprehensiveTest-fail-paged-wrong-pagination-type.graphqls [6:1]: +scalar Long + +type SqrlPagination { +^ + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt index 8908f7aec6..f503b00883 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt @@ -269,6 +269,108 @@ type Query { TableFunctionCallsTblFct(arg1: Int, arg2: Int, limit: Int = 10, offset: Int = 0): [Customer!] } +>>>comprehensiveTest-fail-paged-no-limit-offset.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} + +>>>comprehensiveTest-fail-paged-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} + +>>>comprehensiveTest-fail-paged-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-parameters-names.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -417,6 +519,71 @@ type Query { SelectCustomers(offset: Int = 0): [Customer!] } +>>>comprehensiveTest-paged-results.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-paged-userdefined.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-parameters-order.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt new file mode 100644 index 0000000000..bf34820de8 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -0,0 +1,1819 @@ +>>>comprehensiveTest-fail-duplicate-tablefunction.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-duplicate-tablefunction2.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-mutation.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-query.graphqls +type Query { +} + +>>>comprehensiveTest-fail-empty-result-type.graphqls +type Customer { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-invalid-field-name.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name~: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-invalid-type-name.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer~ { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer~!] +} + +>>>comprehensiveTest-fail-no-query-root-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float +} + +type Subscription { + CustomerSubscription: Customer +} + +>>>comprehensiveTest-fail-no-type.graphqls +type Query { + AnotherCustomer(limit: Int = 10, offset: Int = 0): [AnotherCustomer!] +} + +>>>comprehensiveTest-fail-not-equal-mutations-field-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Int +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-not-equal-mutations-inout-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float + different_output: Float +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-nullableArgForNotNull.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int, arg2: Int, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-paged-no-limit-offset.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} + +>>>comprehensiveTest-fail-paged-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} + +>>>comprehensiveTest-fail-paged-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-parameters-names.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(wrongName: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-parameters-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Float!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-query-with-no-table-function.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct1(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-subscription-with-no-table-function.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + TableFunctionCallsTblFct1(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + unknown: Int +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-wrong-query.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [[Customer!]] +} + +>>>comprehensiveTest-limit-offset-combinations.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + CustomerByTime2: [Customer!] + CustomerByMultipleTime(limit: Int = 10): [Customer!] + SelectCustomers(offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-paged-results.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-paged-userdefined.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-parameters-order.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg2: Int!, arg1: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest.graphqls +type AnotherCustomer { + customerid: Long! + email: String! + lastUpdated: Long! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type CustomerByMultipleTime { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerByTime2 { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerFilteredDistinct { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerTimeWindow { + window_start: DateTime! + window_end: DateTime! + unique_email_count: Long! +} + +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime + +type ExplicitDistinct { + customerid: Long! + timestamp: DateTime + name: String! +} + +type ExternalOrders { + id: Long! + customerid: Long! + time: DateTime! + entries: [ExternalOrders_entriesOutput]! +} + +type ExternalOrders_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +"A 64-bit signed integer" +scalar Long + +type InvalidDistinct { + customerid: Long! + timestamp: DateTime + namee: String! +} + +type MissedTemporalJoin { + id: Long! + customerid: Long! + time: DateTime! + entries: [MissedTemporalJoin_entriesOutput]! + customerid0: Long! + timestamp: DateTime + name: String! +} + +type MissedTemporalJoin_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +type Orders { + orderid: Int! + amount: Float +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float +} + +type Query { + AnotherCustomer(limit: Int = 10, offset: Int = 0): [AnotherCustomer!] + Customer(limit: Int = 10, offset: Int = 0): [Customer!] + CustomerByMultipleTime(limit: Int = 10, offset: Int = 0): [CustomerByMultipleTime!] + CustomerByTime2(limit: Int = 10, offset: Int = 0): [CustomerByTime2!] + CustomerFilteredDistinct(limit: Int = 10, offset: Int = 0): [CustomerFilteredDistinct!] + CustomerTimeWindow(limit: Int = 10, offset: Int = 0): [CustomerTimeWindow!] + ExplicitDistinct(limit: Int = 10, offset: Int = 0): [ExplicitDistinct!] + ExternalOrders(limit: Int = 10, offset: Int = 0): [ExternalOrders!] + InvalidDistinct(limit: Int = 10, offset: Int = 0): [InvalidDistinct!] + MissedTemporalJoin(limit: Int = 10, offset: Int = 0): [MissedTemporalJoin!] + Orders(limit: Int = 10, offset: Int = 0): [Orders!] + """ + This is for selected customers + and their orders + """ + SelectCustomers(limit: Int = 10, offset: Int = 0): [Customer!] + TemporalJoin(limit: Int = 10, offset: Int = 0): [TemporalJoin!] + UnnestOrders(limit: Int = 10, offset: Int = 0): [UnnestOrders!] + CustomerById(minId: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + CustomerQuery(id: Long!, limit: Int = 10, offset: Int = 0): [AnotherCustomer!] + TableFunctionCallsTblFct(arg1: Int, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + CustomerSubscription: Customer + CustomerSubscriptionById(minId: Int!): Customer +} + +type TemporalJoin { + id: Long! + customerid: Long! + time: DateTime! + entries: [TemporalJoin_entriesOutput]! + customerid0: Long! + timestamp: DateTime + name: String! +} + +type TemporalJoin_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +type UnnestOrders { + id: Long! + customerid: Long! + time: DateTime! + productid: Long! + quantity: Long! + discount: Float + newId: Long! +} + +>>>pipeline_explain.txt +=== AnotherCustomer +ID: default_catalog.default_database.AnotherCustomer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: - +Row count: ~5e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL +Inputs: + - default_catalog.default_database._Customer +Annotations: + - stream-root: _Customer + +=== Customer +ID: default_catalog.default_database.Customer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer__base +Annotations: + - stream-root: Customer + +=== CustomerById +ID: default_catalog.default_database.CustomerById +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - parameters: minId + - base-table: Customer + +=== CustomerByMultipleTime +ID: default_catalog.default_database.CustomerByMultipleTime +Type: state +Stage: flink +Primary key: customerid, email +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - mostRecentDistinct: true + - stream-root: Customer + +=== CustomerByTime2 +ID: default_catalog.default_database.CustomerByTime2 +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - mostRecentDistinct: true + - stream-root: Customer + +=== CustomerFilteredDistinct +ID: default_catalog.default_database.CustomerFilteredDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e6 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== CustomerQuery +ID: default_catalog.default_database.CustomerQuery +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.AnotherCustomer +Annotations: + - stream-root: _Customer + - parameters: id + - base-table: AnotherCustomer + +=== CustomerSubscription +ID: default_catalog.default_database.CustomerSubscription +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== CustomerSubscriptionById +ID: default_catalog.default_database.CustomerSubscriptionById +Type: query +Stage: kafka +--- +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - parameters: minId + - base-table: Customer + +=== CustomerTimeWindow +ID: default_catalog.default_database.CustomerTimeWindow +Type: stream +Stage: flink +Primary key: window_start, window_end +Timestamp: - +Row count: ~3e7 +--- +Schema: + - window_start: TIMESTAMP(3) NOT NULL + - window_end: TIMESTAMP(3) NOT NULL + - unique_email_count: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.SelectCustomers +Annotations: + - features: STREAM_WINDOW_AGGREGATION (feature) + +=== ExplicitDistinct +ID: default_catalog.default_database.ExplicitDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== ExternalOrders +ID: default_catalog.default_database.ExternalOrders +Type: stream +Stage: flink +Primary key: id, time +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL +Inputs: + - default_catalog.default_database.ExternalOrders__base +Annotations: + - features: DENORMALIZE (feature) + - stream-root: ExternalOrders + +=== InvalidDistinct +ID: default_catalog.default_database.InvalidDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - namee: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== MissedTemporalJoin +ID: default_catalog.default_database.MissedTemporalJoin +Type: state +Stage: postgres +Primary key: id, time +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL + - customerid0: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.ExplicitDistinct + - default_catalog.default_database.ExternalOrders + +=== Orders +ID: default_catalog.default_database.Orders +Type: state +Stage: flink +Primary key: orderid +Timestamp: - +Row count: ~1e8 +--- +Schema: + - orderid: INTEGER NOT NULL + - amount: FLOAT +Inputs: + - default_catalog.default_database.Orders__base + +=== SelectCustomers +ID: default_catalog.default_database.SelectCustomers +Type: stream +Stage: flink +Primary key: customerid, name +Timestamp: timestamp +Row count: ~5e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - sort: [4 DESC-nulls-last] + +=== TableFunctionCallsTblFct +ID: default_catalog.default_database.TableFunctionCallsTblFct +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.CustomerById +Annotations: + - features: TABLE_FUNCTION_SCAN (feature) + - stream-root: Customer + - parameters: arg1, arg2 + - base-table: Customer + +=== TemporalJoin +ID: default_catalog.default_database.TemporalJoin +Type: stream +Stage: flink +Primary key: id, time +Timestamp: time +Row count: ~9e7 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL + - customerid0: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.ExplicitDistinct + - default_catalog.default_database.ExternalOrders +Annotations: + - stream-root: ExternalOrders + +=== UnnestOrders +ID: default_catalog.default_database.UnnestOrders +Type: stream +Stage: flink +Primary key: - +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - productid: BIGINT NOT NULL + - quantity: BIGINT NOT NULL + - discount: DOUBLE + - newId: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.ExternalOrders +Annotations: + - stream-root: ExternalOrders + +=== _Customer +ID: default_catalog.default_database._Customer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database._Customer__base +Annotations: + - stream-root: _Customer + +=== customersink +ID: mysink.customersink +Type: export +Stage: flink +Connector: print +--- +Inputs: + - default_catalog.default_database.TemporalJoin + +=== TimeWindow +ID: print.TimeWindow +Type: export +Stage: flink +--- +Inputs: + - default_catalog.default_database.CustomerTimeWindow + +>>>flink-sql-no-functions.sql +CREATE TEMPORARY TABLE `Customer__schema` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `Customer` ( + `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `Customer__schema`; +CREATE TEMPORARY TABLE `ExternalOrders__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `ExternalOrders` ( + PRIMARY KEY (`id`, `time`) NOT ENFORCED, + WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `ExternalOrders__schema`; +CREATE TEMPORARY TABLE `_Customer__schema` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Customer` ( + `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Customer__schema`; +CREATE TEMPORARY TABLE `_Orders__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Orders` ( + PRIMARY KEY (`id`, `time`) NOT ENFORCED, + WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Orders__schema`; +CREATE TEMPORARY TABLE `_Product__schema` ( + `productid` BIGINT NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `description` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `category` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `_ingest_time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Product` ( + PRIMARY KEY (`productid`, `name`, `description`, `category`) NOT ENFORCED, + WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Product__schema`; +CREATE VIEW `CustomerByTime2` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC NULLS LAST) AS `__sqrlinternal_rownum` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` +WHERE `__sqrlinternal_rownum` = 1; +CREATE VIEW `CustomerFilteredDistinct` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `lastUpdated` DESC NULLS LAST) AS `$f5` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, LAG(`email`, 1) OVER (PARTITION BY `customerid` ORDER BY `timestamp`) AS `$f5`, LAG(`name`, 1) OVER (PARTITION BY `customerid` ORDER BY `timestamp`) AS `$f6` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, MAX(`lastUpdated`) OVER (PARTITION BY `customerid` ORDER BY `timestamp` RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `$f5` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` + WHERE `lastUpdated` >= `$f5`) AS `t1` + WHERE `$f5` IS NULL AND `$f6` IS NULL OR `email` <> `$f5` OR `name` <> `$f6`) AS `t3`) AS `t4` +WHERE `$f5` = 1; +CREATE VIEW `AnotherCustomer` +AS +SELECT `customerid`, `email`, `lastUpdated` +FROM `_Customer` +WHERE `customerid` > 100; +CREATE VIEW `CustomerByMultipleTime` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid`, `email` ORDER BY `timestamp` DESC NULLS LAST, `lastUpdated` NULLS FIRST) AS `__sqrlinternal_rownum` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` +WHERE `__sqrlinternal_rownum` = 1; +CREATE VIEW `ExplicitDistinct` +AS +SELECT `customerid`, `timestamp`, `name` +FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC) AS `_rownum` + FROM `Customer`) +WHERE `_rownum` = 1; +CREATE VIEW `InvalidDistinct` +AS +SELECT `customerid`, `timestamp`, `name` AS `namee` +FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC) AS `_rownum` + FROM `Customer`) +WHERE `_rownum` = 1; +CREATE VIEW `MissedTemporalJoin` +AS +SELECT * +FROM `ExternalOrders` AS `o` + INNER JOIN `ExplicitDistinct` AS `c` ON `o`.`customerid` = `c`.`customerid`; +CREATE VIEW `TemporalJoin` +AS +SELECT * +FROM `ExternalOrders` AS `o` + INNER JOIN `ExplicitDistinct` FOR SYSTEM_TIME AS OF `time` AS `c` ON `o`.`customerid` = `c`.`customerid`; +CREATE VIEW `SelectCustomers` +AS +SELECT * +FROM `Customer` +WHERE `customerid` > 0; +CREATE VIEW `CustomerSubscription` +AS +SELECT * +FROM `Customer`; +CREATE VIEW `UnnestOrders` +AS +SELECT `o`.`id`, `o`.`customerid`, `o`.`time`, `e`.`productid`, `e`.`quantity`, `e`.`discount` +FROM `ExternalOrders` AS `o` + CROSS JOIN UNNEST(`entries`) AS `e`; +ALTER VIEW `UnnestOrders` +AS +SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `id` + `productid` AS `newId` +FROM (SELECT `o`.`id`, `o`.`customerid`, `o`.`time`, `e`.`productid`, `e`.`quantity`, `e`.`discount` + FROM `ExternalOrders` AS `o` + CROSS JOIN UNNEST(`entries`) AS `e`); +CREATE TABLE `Orders` ( + `orderid` INTEGER, + `amount` FLOAT, + PRIMARY KEY (`orderid`) NOT ENFORCED +) +WITH ( + 'connector' = 'upsert-kafka', + 'key.format' = 'flexible-json', + 'properties.auto.offset.reset' = 'earliest', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'Orders', + 'value.fields-include' = 'ALL', + 'value.format' = 'flexible-json' +); +CREATE VIEW `CustomerTimeWindow` +AS +SELECT `window_start`, `window_end`, COUNT(DISTINCT `email`) AS `unique_email_count` +FROM TABLE(TUMBLE(TABLE `SelectCustomers`, DESCRIPTOR(`timestamp`), INTERVAL '1' MINUTE)) +GROUP BY `window_start`, `window_end`; +CREATE TEMPORARY TABLE `MyPrintSink_ex1__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, + `customerid0` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `MyPrintSink_ex1` ( + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'print' +) +LIKE `MyPrintSink_ex1__schema`; +CREATE TABLE `AnotherCustomer_1` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'AnotherCustomer', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Customer_2` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'Customer', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Customer_3` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'kafka', + 'format' = 'flexible-json', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'Customer' +); +CREATE TABLE `CustomerByMultipleTime_4` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `email`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerByMultipleTime', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerByTime2_5` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerByTime2', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerFilteredDistinct_6` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerFilteredDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerSubscription_7` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'kafka', + 'format' = 'flexible-json', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'CustomerSubscription' +); +CREATE TABLE `TimeWindow_8` ( + `window_start` TIMESTAMP(3) NOT NULL, + `window_end` TIMESTAMP(3) NOT NULL, + `unique_email_count` BIGINT NOT NULL, + PRIMARY KEY (`window_start`, `window_end`) NOT ENFORCED +) +WITH ( + 'connector' = 'print', + 'print-identifier' = 'TimeWindow' +); +CREATE TABLE `CustomerTimeWindow_9` ( + `window_start` TIMESTAMP(3) NOT NULL, + `window_end` TIMESTAMP(3) NOT NULL, + `unique_email_count` BIGINT NOT NULL, + PRIMARY KEY (`window_start`, `window_end`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'CustomerTimeWindow', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `ExplicitDistinct_10` ( + `customerid` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'ExplicitDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `ExternalOrders_11` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` RAW('com.datasqrl.flinkrunner.stdlib.json.FlinkJsonType', 'AERjb20uZGF0YXNxcmwuZmxpbmtydW5uZXIuc3RkbGliLmpzb24uRmxpbmtKc29uVHlwZVNlcmlhbGl6ZXJTbmFwc2hvdAAAAAM='), + PRIMARY KEY (`id`, `time`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'ExternalOrders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `InvalidDistinct_12` ( + `customerid` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `namee` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'InvalidDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Orders_13` ( + `orderid` INTEGER NOT NULL, + `amount` FLOAT, + PRIMARY KEY (`orderid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'table-name' = 'Orders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `SelectCustomers_14` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `name`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'SelectCustomers', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `TemporalJoin_15` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` RAW('com.datasqrl.flinkrunner.stdlib.json.FlinkJsonType', 'AERjb20uZGF0YXNxcmwuZmxpbmtydW5uZXIuc3RkbGliLmpzb24uRmxpbmtKc29uVHlwZVNlcmlhbGl6ZXJTbmFwc2hvdAAAAAM='), + `customerid0` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`id`, `time`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'TemporalJoin', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `UnnestOrders_16` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `productid` BIGINT NOT NULL, + `quantity` BIGINT NOT NULL, + `discount` DOUBLE, + `newId` BIGINT NOT NULL, + `__pk_hash` CHAR(32) CHARACTER SET `UTF-16LE`, + PRIMARY KEY (`__pk_hash`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'UnnestOrders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +EXECUTE STATEMENT SET BEGIN +INSERT INTO `default_catalog`.`default_database`.`AnotherCustomer_1` +SELECT * + FROM `default_catalog`.`default_database`.`AnotherCustomer` +; +INSERT INTO `default_catalog`.`default_database`.`Customer_2` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`Customer_3` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerSubscription` + ; + INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerTimeWindow` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerTimeWindow` + ; + INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` + SELECT * + FROM `default_catalog`.`default_database`.`ExplicitDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` + SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` + FROM `default_catalog`.`default_database`.`ExternalOrders` + ; + INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` + SELECT * + FROM `default_catalog`.`default_database`.`InvalidDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`Orders_13` + SELECT * + FROM `default_catalog`.`default_database`.`Orders` + ; + INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` + SELECT * + FROM `default_catalog`.`default_database`.`SelectCustomers` + ; + INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` + SELECT * + FROM `default_catalog`.`default_database`.`TemporalJoin` + ; + INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` + SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` + FROM `default_catalog`.`default_database`.`TemporalJoin` + ; + INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` + SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` + FROM `default_catalog`.`default_database`.`UnnestOrders` + ; + END +>>>kafka.json +{ + "topics" : [ + { + "topicName" : "Customer", + "tableName" : "Customer_3", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "SUBSCRIPTION", + "messageKeys" : [ ], + "messageSchema" : "", + "config" : { } + }, + { + "topicName" : "CustomerSubscription", + "tableName" : "CustomerSubscription_7", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "SUBSCRIPTION", + "messageKeys" : [ ], + "messageSchema" : "", + "config" : { } + }, + { + "topicName" : "Orders", + "tableName" : "Orders", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "MUTATION", + "messageKeys" : [ + "orderid" + ], + "messageSchema" : "", + "config" : { + "cleanup.policy" : "compact", + "retention.ms" : "129600000" + } + } + ], + "testRunnerTopics" : [ ] +} +>>>postgres-schema.sql +CREATE TABLE IF NOT EXISTS "AnotherCustomer" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, PRIMARY KEY ("customerid","lastUpdated")); +CREATE TABLE IF NOT EXISTS "Customer" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","lastUpdated")); +CREATE TABLE IF NOT EXISTS "CustomerByMultipleTime" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","email")); +CREATE TABLE IF NOT EXISTS "CustomerByTime2" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "CustomerFilteredDistinct" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "CustomerTimeWindow" ("window_start" TIMESTAMP WITHOUT TIME ZONE NOT NULL, "window_end" TIMESTAMP WITHOUT TIME ZONE NOT NULL, "unique_email_count" BIGINT NOT NULL, PRIMARY KEY ("window_start","window_end")); +CREATE TABLE IF NOT EXISTS "ExplicitDistinct" ("customerid" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "ExternalOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, PRIMARY KEY ("id","time")); +CREATE TABLE IF NOT EXISTS "InvalidDistinct" ("customerid" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "namee" TEXT NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "Orders" ("orderid" INTEGER NOT NULL, "amount" FLOAT, PRIMARY KEY ("orderid")); +CREATE TABLE IF NOT EXISTS "SelectCustomers" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","name")); +CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, "customerid0" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("id","time")); +CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); + +CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name") +>>>postgres-views.sql +CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * +FROM "ExternalOrders" AS "ExternalOrders0" + INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" +>>>vertx.json +{ + "models" : { + "v1" : { + "queries" : [ + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "CustomerByTime2", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM \"CustomerByTime2\"", + "parameters" : [ ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" + } + } + } + ], + "mutations" : [ ], + "subscriptions" : [ ], + "operations" : [ + { + "function" : { + "name" : "GetCustomerByTime2", + "parameters" : { + "type" : "object", + "properties" : { + "offset" : { + "type" : "integer" + }, + "limit" : { + "type" : "integer" + } + }, + "required" : [ ] + } + }, + "format" : "JSON", + "apiQuery" : { + "query" : "query CustomerByTime2($limit: Int = 10, $offset: Int = 0) {\nCustomerByTime2(limit: $limit, offset: $offset) {\nresults {\ncustomerid\nemail\nname\nlastUpdated\ntimestamp\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "queryName" : "CustomerByTime2", + "operationType" : "QUERY" + }, + "mcpMethod" : "TOOL", + "restMethod" : "GET", + "uriTemplate" : "queries/CustomerByTime2{?offset,limit}" + } + ], + "schema" : { + "type" : "string", + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: SqrlPagination\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n\ntype SqrlPagination {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n" + } + } + } +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt new file mode 100644 index 0000000000..2106f83733 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -0,0 +1,1842 @@ +>>>comprehensiveTest-fail-duplicate-tablefunction.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-duplicate-tablefunction2.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-mutation.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-query.graphqls +type Query { +} + +>>>comprehensiveTest-fail-empty-result-type.graphqls +type Customer { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-empty-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-invalid-field-name.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name~: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-invalid-type-name.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer~ { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer~!] +} + +>>>comprehensiveTest-fail-no-query-root-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float +} + +type Subscription { + CustomerSubscription: Customer +} + +>>>comprehensiveTest-fail-no-type.graphqls +type Query { + AnotherCustomer(limit: Int = 10, offset: Int = 0): [AnotherCustomer!] +} + +>>>comprehensiveTest-fail-not-equal-mutations-field-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Int +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-not-equal-mutations-inout-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float + different_output: Float +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-nullableArgForNotNull.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int, arg2: Int, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-paged-no-limit-offset.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} + +>>>comprehensiveTest-fail-paged-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} + +>>>comprehensiveTest-fail-paged-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-parameters-names.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(wrongName: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-parameters-types.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Float!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-query-with-no-table-function.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct1(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-subscription-with-no-table-function.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + TableFunctionCallsTblFct1(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + unknown: Int +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-fail-wrong-query.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [[Customer!]] +} + +>>>comprehensiveTest-limit-offset-combinations.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + CustomerByTime2: [Customer!] + CustomerByMultipleTime(limit: Int = 10): [Customer!] + SelectCustomers(offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest-paged-results.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-paged-userdefined.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-parameters-order.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Query { + TableFunctionCallsTblFct(arg2: Int!, arg1: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +>>>comprehensiveTest.graphqls +type AnotherCustomer { + customerid: Long! + email: String! + lastUpdated: Long! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime + related(limit: Int = 10, offset: Int = 0): [Customer!] + relatedByLength(length: Int, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type CustomerByMultipleTime { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerByTime2 { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerFilteredDistinct { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime +} + +type CustomerTimeWindow { + window_start: DateTime! + window_end: DateTime! + unique_email_count: Long! +} + +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime + +type ExplicitDistinct { + customerid: Long! + timestamp: DateTime + name: String! +} + +type ExternalOrders { + id: Long! + customerid: Long! + time: DateTime! + entries: [ExternalOrders_entriesOutput]! +} + +type ExternalOrders_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +"A 64-bit signed integer" +scalar Long + +type InvalidDistinct { + customerid: Long! + timestamp: DateTime + namee: String! +} + +type MissedTemporalJoin { + id: Long! + customerid: Long! + time: DateTime! + entries: [MissedTemporalJoin_entriesOutput]! + customerid0: Long! + timestamp: DateTime + name: String! +} + +type MissedTemporalJoin_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +type Mutation { + Orders(event: OrdersInput!): OrdersResultOutput! +} + +type Orders { + orderid: Int! + amount: Float +} + +input OrdersInput { + orderid: Int! + amount: Float +} + +type OrdersResultOutput { + orderid: Int! + amount: Float +} + +type Query { + AnotherCustomer(limit: Int = 10, offset: Int = 0): [AnotherCustomer!] + Customer(limit: Int = 10, offset: Int = 0): [Customer!] + CustomerByMultipleTime(limit: Int = 10, offset: Int = 0): [CustomerByMultipleTime!] + CustomerByTime2(limit: Int = 10, offset: Int = 0): [CustomerByTime2!] + CustomerFilteredDistinct(limit: Int = 10, offset: Int = 0): [CustomerFilteredDistinct!] + CustomerTimeWindow(limit: Int = 10, offset: Int = 0): [CustomerTimeWindow!] + ExplicitDistinct(limit: Int = 10, offset: Int = 0): [ExplicitDistinct!] + ExternalOrders(limit: Int = 10, offset: Int = 0): [ExternalOrders!] + InvalidDistinct(limit: Int = 10, offset: Int = 0): [InvalidDistinct!] + MissedTemporalJoin(limit: Int = 10, offset: Int = 0): [MissedTemporalJoin!] + Orders(limit: Int = 10, offset: Int = 0): [Orders!] + """ + This is for selected customers + and their orders + """ + SelectCustomers(limit: Int = 10, offset: Int = 0): [Customer!] + TemporalJoin(limit: Int = 10, offset: Int = 0): [TemporalJoin!] + UnnestOrders(limit: Int = 10, offset: Int = 0): [UnnestOrders!] + CustomerById(minId: Int!, limit: Int = 10, offset: Int = 0): [Customer!] + CustomerQuery(id: Long!, limit: Int = 10, offset: Int = 0): [AnotherCustomer!] + TableFunctionCallsTblFct(arg1: Int, arg2: Int!, limit: Int = 10, offset: Int = 0): [Customer!] +} + +type Subscription { + CustomerSubscription: Customer + CustomerSubscriptionById(minId: Int!): Customer +} + +type TemporalJoin { + id: Long! + customerid: Long! + time: DateTime! + entries: [TemporalJoin_entriesOutput]! + customerid0: Long! + timestamp: DateTime + name: String! +} + +type TemporalJoin_entriesOutput { + productid: Long! + quantity: Long! + unit_price: Float! + discount: Float +} + +type UnnestOrders { + id: Long! + customerid: Long! + time: DateTime! + productid: Long! + quantity: Long! + discount: Float + newId: Long! +} + +>>>pipeline_explain.txt +=== AnotherCustomer +ID: default_catalog.default_database.AnotherCustomer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: - +Row count: ~5e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL +Inputs: + - default_catalog.default_database._Customer +Annotations: + - stream-root: _Customer + +=== Customer +ID: default_catalog.default_database.Customer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer__base +Annotations: + - stream-root: Customer + +=== CustomerById +ID: default_catalog.default_database.CustomerById +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - parameters: minId + - base-table: Customer + +=== CustomerByMultipleTime +ID: default_catalog.default_database.CustomerByMultipleTime +Type: state +Stage: flink +Primary key: customerid, email +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - mostRecentDistinct: true + - stream-root: Customer + +=== CustomerByTime2 +ID: default_catalog.default_database.CustomerByTime2 +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - mostRecentDistinct: true + - stream-root: Customer + +=== CustomerFilteredDistinct +ID: default_catalog.default_database.CustomerFilteredDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e6 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== CustomerQuery +ID: default_catalog.default_database.CustomerQuery +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.AnotherCustomer +Annotations: + - stream-root: _Customer + - parameters: id + - base-table: AnotherCustomer + +=== CustomerSubscription +ID: default_catalog.default_database.CustomerSubscription +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== CustomerSubscriptionById +ID: default_catalog.default_database.CustomerSubscriptionById +Type: query +Stage: kafka +--- +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - parameters: minId + - base-table: Customer + +=== CustomerTimeWindow +ID: default_catalog.default_database.CustomerTimeWindow +Type: stream +Stage: flink +Primary key: window_start, window_end +Timestamp: - +Row count: ~3e7 +--- +Schema: + - window_start: TIMESTAMP(3) NOT NULL + - window_end: TIMESTAMP(3) NOT NULL + - unique_email_count: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.SelectCustomers +Annotations: + - features: STREAM_WINDOW_AGGREGATION (feature) + +=== ExplicitDistinct +ID: default_catalog.default_database.ExplicitDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== ExternalOrders +ID: default_catalog.default_database.ExternalOrders +Type: stream +Stage: flink +Primary key: id, time +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL +Inputs: + - default_catalog.default_database.ExternalOrders__base +Annotations: + - features: DENORMALIZE (feature) + - stream-root: ExternalOrders + +=== InvalidDistinct +ID: default_catalog.default_database.InvalidDistinct +Type: state +Stage: flink +Primary key: customerid +Timestamp: timestamp +Row count: ~2e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - namee: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + +=== MissedTemporalJoin +ID: default_catalog.default_database.MissedTemporalJoin +Type: state +Stage: postgres +Primary key: id, time +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL + - customerid0: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.ExplicitDistinct + - default_catalog.default_database.ExternalOrders + +=== Orders +ID: default_catalog.default_database.Orders +Type: state +Stage: flink +Primary key: orderid +Timestamp: - +Row count: ~1e8 +--- +Schema: + - orderid: INTEGER NOT NULL + - amount: FLOAT +Inputs: + - default_catalog.default_database.Orders__base + +=== SelectCustomers +ID: default_catalog.default_database.SelectCustomers +Type: stream +Stage: flink +Primary key: customerid, name +Timestamp: timestamp +Row count: ~5e7 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Customer +Annotations: + - stream-root: Customer + - sort: [4 DESC-nulls-last] + +=== TableFunctionCallsTblFct +ID: default_catalog.default_database.TableFunctionCallsTblFct +Type: query +Stage: postgres +--- +Inputs: + - default_catalog.default_database.CustomerById +Annotations: + - features: TABLE_FUNCTION_SCAN (feature) + - stream-root: Customer + - parameters: arg1, arg2 + - base-table: Customer + +=== TemporalJoin +ID: default_catalog.default_database.TemporalJoin +Type: stream +Stage: flink +Primary key: id, time +Timestamp: time +Row count: ~9e7 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - entries: RecordType:peek_no_expand(BIGINT NOT NULL productid, BIGINT NOT NULL quantity, DOUBLE NOT NULL unit_price, DOUBLE discount) NOT NULL ARRAY NOT NULL + - customerid0: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.ExplicitDistinct + - default_catalog.default_database.ExternalOrders +Annotations: + - stream-root: ExternalOrders + +=== UnnestOrders +ID: default_catalog.default_database.UnnestOrders +Type: stream +Stage: flink +Primary key: - +Timestamp: time +Row count: ~1e8 +--- +Schema: + - id: BIGINT NOT NULL + - customerid: BIGINT NOT NULL + - time: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - productid: BIGINT NOT NULL + - quantity: BIGINT NOT NULL + - discount: DOUBLE + - newId: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.ExternalOrders +Annotations: + - stream-root: ExternalOrders + +=== _Customer +ID: default_catalog.default_database._Customer +Type: stream +Stage: flink +Primary key: customerid, lastUpdated +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - customerid: BIGINT NOT NULL + - email: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - name: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - lastUpdated: BIGINT NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database._Customer__base +Annotations: + - stream-root: _Customer + +=== customersink +ID: mysink.customersink +Type: export +Stage: flink +Connector: print +--- +Inputs: + - default_catalog.default_database.TemporalJoin + +=== TimeWindow +ID: print.TimeWindow +Type: export +Stage: flink +--- +Inputs: + - default_catalog.default_database.CustomerTimeWindow + +>>>flink-sql-no-functions.sql +CREATE TEMPORARY TABLE `Customer__schema` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `Customer` ( + `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `Customer__schema`; +CREATE TEMPORARY TABLE `ExternalOrders__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `ExternalOrders` ( + PRIMARY KEY (`id`, `time`) NOT ENFORCED, + WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `ExternalOrders__schema`; +CREATE TEMPORARY TABLE `_Customer__schema` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Customer` ( + `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Customer__schema`; +CREATE TEMPORARY TABLE `_Orders__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Orders` ( + PRIMARY KEY (`id`, `time`) NOT ENFORCED, + WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Orders__schema`; +CREATE TEMPORARY TABLE `_Product__schema` ( + `productid` BIGINT NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `description` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `category` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `_ingest_time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `_Product` ( + PRIMARY KEY (`productid`, `name`, `description`, `category`) NOT ENFORCED, + WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = 'file:/mock', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `_Product__schema`; +CREATE VIEW `CustomerByTime2` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC NULLS LAST) AS `__sqrlinternal_rownum` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` +WHERE `__sqrlinternal_rownum` = 1; +CREATE VIEW `CustomerFilteredDistinct` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `lastUpdated` DESC NULLS LAST) AS `$f5` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, LAG(`email`, 1) OVER (PARTITION BY `customerid` ORDER BY `timestamp`) AS `$f5`, LAG(`name`, 1) OVER (PARTITION BY `customerid` ORDER BY `timestamp`) AS `$f6` + FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, MAX(`lastUpdated`) OVER (PARTITION BY `customerid` ORDER BY `timestamp` RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `$f5` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` + WHERE `lastUpdated` >= `$f5`) AS `t1` + WHERE `$f5` IS NULL AND `$f6` IS NULL OR `email` <> `$f5` OR `name` <> `$f6`) AS `t3`) AS `t4` +WHERE `$f5` = 1; +CREATE VIEW `AnotherCustomer` +AS +SELECT `customerid`, `email`, `lastUpdated` +FROM `_Customer` +WHERE `customerid` > 100; +CREATE VIEW `CustomerByMultipleTime` +AS +SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` +FROM (SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp`, ROW_NUMBER() OVER (PARTITION BY `customerid`, `email` ORDER BY `timestamp` DESC NULLS LAST, `lastUpdated` NULLS FIRST) AS `__sqrlinternal_rownum` + FROM `default_catalog`.`default_database`.`Customer`) AS `t` +WHERE `__sqrlinternal_rownum` = 1; +CREATE VIEW `ExplicitDistinct` +AS +SELECT `customerid`, `timestamp`, `name` +FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC) AS `_rownum` + FROM `Customer`) +WHERE `_rownum` = 1; +CREATE VIEW `InvalidDistinct` +AS +SELECT `customerid`, `timestamp`, `name` AS `namee` +FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY `customerid` ORDER BY `timestamp` DESC) AS `_rownum` + FROM `Customer`) +WHERE `_rownum` = 1; +CREATE VIEW `MissedTemporalJoin` +AS +SELECT * +FROM `ExternalOrders` AS `o` + INNER JOIN `ExplicitDistinct` AS `c` ON `o`.`customerid` = `c`.`customerid`; +CREATE VIEW `TemporalJoin` +AS +SELECT * +FROM `ExternalOrders` AS `o` + INNER JOIN `ExplicitDistinct` FOR SYSTEM_TIME AS OF `time` AS `c` ON `o`.`customerid` = `c`.`customerid`; +CREATE VIEW `SelectCustomers` +AS +SELECT * +FROM `Customer` +WHERE `customerid` > 0; +CREATE VIEW `CustomerSubscription` +AS +SELECT * +FROM `Customer`; +CREATE VIEW `UnnestOrders` +AS +SELECT `o`.`id`, `o`.`customerid`, `o`.`time`, `e`.`productid`, `e`.`quantity`, `e`.`discount` +FROM `ExternalOrders` AS `o` + CROSS JOIN UNNEST(`entries`) AS `e`; +ALTER VIEW `UnnestOrders` +AS +SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `id` + `productid` AS `newId` +FROM (SELECT `o`.`id`, `o`.`customerid`, `o`.`time`, `e`.`productid`, `e`.`quantity`, `e`.`discount` + FROM `ExternalOrders` AS `o` + CROSS JOIN UNNEST(`entries`) AS `e`); +CREATE TABLE `Orders` ( + `orderid` INTEGER, + `amount` FLOAT, + PRIMARY KEY (`orderid`) NOT ENFORCED +) +WITH ( + 'connector' = 'upsert-kafka', + 'key.format' = 'flexible-json', + 'properties.auto.offset.reset' = 'earliest', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'Orders', + 'value.fields-include' = 'ALL', + 'value.format' = 'flexible-json' +); +CREATE VIEW `CustomerTimeWindow` +AS +SELECT `window_start`, `window_end`, COUNT(DISTINCT `email`) AS `unique_email_count` +FROM TABLE(TUMBLE(TABLE `SelectCustomers`, DESCRIPTOR(`timestamp`), INTERVAL '1' MINUTE)) +GROUP BY `window_start`, `window_end`; +CREATE TEMPORARY TABLE `MyPrintSink_ex1__schema` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, + `customerid0` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `MyPrintSink_ex1` ( + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'print' +) +LIKE `MyPrintSink_ex1__schema`; +CREATE TABLE `AnotherCustomer_1` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'AnotherCustomer', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Customer_2` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'Customer', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Customer_3` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'kafka', + 'format' = 'flexible-json', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'Customer' +); +CREATE TABLE `CustomerByMultipleTime_4` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `email`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerByMultipleTime', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerByTime2_5` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerByTime2', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerFilteredDistinct_6` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'CustomerFilteredDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `CustomerSubscription_7` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL +) +WITH ( + 'connector' = 'kafka', + 'format' = 'flexible-json', + 'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}', + 'properties.compression.type' = 'zstd', + 'properties.group.id' = '${KAFKA_GROUP_ID}', + 'topic' = 'CustomerSubscription' +); +CREATE TABLE `TimeWindow_8` ( + `window_start` TIMESTAMP(3) NOT NULL, + `window_end` TIMESTAMP(3) NOT NULL, + `unique_email_count` BIGINT NOT NULL, + PRIMARY KEY (`window_start`, `window_end`) NOT ENFORCED +) +WITH ( + 'connector' = 'print', + 'print-identifier' = 'TimeWindow' +); +CREATE TABLE `CustomerTimeWindow_9` ( + `window_start` TIMESTAMP(3) NOT NULL, + `window_end` TIMESTAMP(3) NOT NULL, + `unique_email_count` BIGINT NOT NULL, + PRIMARY KEY (`window_start`, `window_end`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'CustomerTimeWindow', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `ExplicitDistinct_10` ( + `customerid` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'ExplicitDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `ExternalOrders_11` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` RAW('com.datasqrl.flinkrunner.stdlib.json.FlinkJsonType', 'AERjb20uZGF0YXNxcmwuZmxpbmtydW5uZXIuc3RkbGliLmpzb24uRmxpbmtKc29uVHlwZVNlcmlhbGl6ZXJTbmFwc2hvdAAAAAM='), + PRIMARY KEY (`id`, `time`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'ExternalOrders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `InvalidDistinct_12` ( + `customerid` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `namee` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`customerid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'TIMESTAMP', + 'sink.on-conflict.timestamp-column' = 'timestamp', + 'table-name' = 'InvalidDistinct', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Orders_13` ( + `orderid` INTEGER NOT NULL, + `amount` FLOAT, + PRIMARY KEY (`orderid`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'table-name' = 'Orders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `SelectCustomers_14` ( + `customerid` BIGINT NOT NULL, + `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `lastUpdated` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + PRIMARY KEY (`customerid`, `name`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'SelectCustomers', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `TemporalJoin_15` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `entries` RAW('com.datasqrl.flinkrunner.stdlib.json.FlinkJsonType', 'AERjb20uZGF0YXNxcmwuZmxpbmtydW5uZXIuc3RkbGliLmpzb24uRmxpbmtKc29uVHlwZVNlcmlhbGl6ZXJTbmFwc2hvdAAAAAM='), + `customerid0` BIGINT NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`id`, `time`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'TemporalJoin', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `UnnestOrders_16` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `productid` BIGINT NOT NULL, + `quantity` BIGINT NOT NULL, + `discount` DOUBLE, + `newId` BIGINT NOT NULL, + `__pk_hash` CHAR(32) CHARACTER SET `UTF-16LE`, + PRIMARY KEY (`__pk_hash`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'UnnestOrders', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +EXECUTE STATEMENT SET BEGIN +INSERT INTO `default_catalog`.`default_database`.`AnotherCustomer_1` +SELECT * + FROM `default_catalog`.`default_database`.`AnotherCustomer` +; +INSERT INTO `default_catalog`.`default_database`.`Customer_2` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`Customer_3` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` + SELECT * + FROM `default_catalog`.`default_database`.`Customer` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerSubscription` + ; + INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerTimeWindow` + ; + INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` + SELECT * + FROM `default_catalog`.`default_database`.`CustomerTimeWindow` + ; + INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` + SELECT * + FROM `default_catalog`.`default_database`.`ExplicitDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` + SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` + FROM `default_catalog`.`default_database`.`ExternalOrders` + ; + INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` + SELECT * + FROM `default_catalog`.`default_database`.`InvalidDistinct` + ; + INSERT INTO `default_catalog`.`default_database`.`Orders_13` + SELECT * + FROM `default_catalog`.`default_database`.`Orders` + ; + INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` + SELECT * + FROM `default_catalog`.`default_database`.`SelectCustomers` + ; + INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` + SELECT * + FROM `default_catalog`.`default_database`.`TemporalJoin` + ; + INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` + SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` + FROM `default_catalog`.`default_database`.`TemporalJoin` + ; + INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` + SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` + FROM `default_catalog`.`default_database`.`UnnestOrders` + ; + END +>>>kafka.json +{ + "topics" : [ + { + "topicName" : "Customer", + "tableName" : "Customer_3", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "SUBSCRIPTION", + "messageKeys" : [ ], + "messageSchema" : "", + "config" : { } + }, + { + "topicName" : "CustomerSubscription", + "tableName" : "CustomerSubscription_7", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "SUBSCRIPTION", + "messageKeys" : [ ], + "messageSchema" : "", + "config" : { } + }, + { + "topicName" : "Orders", + "tableName" : "Orders", + "format" : "flexible-json", + "numPartitions" : 1, + "replicationFactor" : 3, + "type" : "MUTATION", + "messageKeys" : [ + "orderid" + ], + "messageSchema" : "", + "config" : { + "cleanup.policy" : "compact", + "retention.ms" : "129600000" + } + } + ], + "testRunnerTopics" : [ ] +} +>>>postgres-schema.sql +CREATE TABLE IF NOT EXISTS "AnotherCustomer" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, PRIMARY KEY ("customerid","lastUpdated")); +CREATE TABLE IF NOT EXISTS "Customer" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","lastUpdated")); +CREATE TABLE IF NOT EXISTS "CustomerByMultipleTime" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","email")); +CREATE TABLE IF NOT EXISTS "CustomerByTime2" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "CustomerFilteredDistinct" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "CustomerTimeWindow" ("window_start" TIMESTAMP WITHOUT TIME ZONE NOT NULL, "window_end" TIMESTAMP WITHOUT TIME ZONE NOT NULL, "unique_email_count" BIGINT NOT NULL, PRIMARY KEY ("window_start","window_end")); +CREATE TABLE IF NOT EXISTS "ExplicitDistinct" ("customerid" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "ExternalOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, PRIMARY KEY ("id","time")); +CREATE TABLE IF NOT EXISTS "InvalidDistinct" ("customerid" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "namee" TEXT NOT NULL, PRIMARY KEY ("customerid")); +CREATE TABLE IF NOT EXISTS "Orders" ("orderid" INTEGER NOT NULL, "amount" FLOAT, PRIMARY KEY ("orderid")); +CREATE TABLE IF NOT EXISTS "SelectCustomers" ("customerid" BIGINT NOT NULL, "email" TEXT NOT NULL, "name" TEXT NOT NULL, "lastUpdated" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, PRIMARY KEY ("customerid","name")); +CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, "customerid0" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("id","time")); +CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); + +CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name") +>>>postgres-views.sql +CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * +FROM "ExternalOrders" AS "ExternalOrders0" + INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" +>>>vertx.json +{ + "models" : { + "v1" : { + "queries" : [ + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "TableFunctionCallsTblFct", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + }, + { + "type" : "variable", + "path" : "arg2" + }, + { + "type" : "variable", + "path" : "arg1" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1", + "parameters" : [ + { + "type" : "arg", + "path" : "arg1", + "sqlType" : "INTEGER" + }, + { + "type" : "arg", + "path" : "arg2", + "sqlType" : "INTEGER" + } + ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" + } + } + }, + { + "type" : "args", + "parentType" : "Customer", + "fieldName" : "related", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST", + "parameters" : [ + { + "type" : "source", + "key" : "customerid" + } + ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" + } + } + } + ], + "mutations" : [ ], + "subscriptions" : [ ], + "operations" : [ ], + "schema" : { + "type" : "string", + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype SqrlPagination {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: SqrlPagination\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: SqrlPagination\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + } + } + } +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt index a1958f76ac..bdcd2dbc4f 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt @@ -269,6 +269,108 @@ type Query { TableFunctionCallsTblFct(arg1: Int, arg2: Int, limit: Int = 10, offset: Int = 0): [Customer!] } +>>>comprehensiveTest-fail-paged-no-limit-offset.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} + +>>>comprehensiveTest-fail-paged-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} + +>>>comprehensiveTest-fail-paged-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-parameters-names.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -417,6 +519,71 @@ type Query { SelectCustomers(offset: Int = 0): [Customer!] } +>>>comprehensiveTest-paged-results.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-paged-userdefined.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-parameters-order.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt index 8df66faad5..e9414ac163 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt @@ -269,6 +269,108 @@ type Query { TableFunctionCallsTblFct(arg1: Int, arg2: Int, limit: Int = 10, offset: Int = 0): [Customer!] } +>>>comprehensiveTest-fail-paged-no-limit-offset.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2: CustomerPage! +} + +>>>comprehensiveTest-fail-paged-subscription.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +type Subscription { + CustomerSubscription: CustomerPage +} + +>>>comprehensiveTest-fail-paged-unknown-field.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + doesNotExist: String! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-parameters-names.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -417,6 +519,71 @@ type Query { SelectCustomers(offset: Int = 0): [Customer!] } +>>>comprehensiveTest-paged-results.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: SqrlPagination +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + +>>>comprehensiveTest-paged-userdefined.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type SqrlPagination { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! + related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage +} + +type CustomerRelatedPage { + items: [Customer!] + meta: SqrlPagination +} + +type CustomerPage { + items: [Customer!] + meta: SqrlPagination +} + +type Query { + TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-parameters-order.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime From 33fa40aaf7d424924383fc949c50a8df745d9dbe Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 8 Jul 2026 11:35:39 -0300 Subject: [PATCH 02/16] feat: Rename pagination type to OffsetPageInfo and compute metadata lazily from the selection set Signed-off-by: Marvin Froeder --- .../datasqrl/config/GraphqlSourceLoader.java | 4 +- .../server/GraphqlModelGenerator.java | 47 +++-- .../server/GraphqlSchemaValidator.java | 2 +- .../datasqrl/server/GraphqlSchemaWalker.java | 4 +- ...ationUtil.java => OffsetPageInfoUtil.java} | 16 +- .../server/graphql/RootGraphQLModel.java | 26 ++- .../datasqrl/server/jdbc/VertxJdbcClient.java | 8 +- .../jdbc/VertxQueryExecutionContext.java | 151 +++++++++++----- .../java/com/datasqrl/server/WriteIT.java | 1 + .../server/jdbc/PaginationMetadataTest.java | 44 ++++- ...veTest-fail-paged-no-limit-offset.graphqls | 2 +- ...nsiveTest-fail-paged-subscription.graphqls | 2 +- ...siveTest-fail-paged-unknown-field.graphqls | 2 +- ...-fail-paged-wrong-pagination-type.graphqls | 4 +- .../comprehensiveTest-paged-results.graphqls | 2 +- ...mprehensiveTest-paged-userdefined.graphqls | 6 +- ...eTest-fail-paged-wrong-pagination-type.txt | 6 +- ...ehensiveTest-limit-offset-combinations.txt | 18 +- .../comprehensiveTest-paged-results.txt | 160 ++++++++--------- .../comprehensiveTest-paged-userdefined.txt | 163 +++++++++--------- .../comprehensiveTest-parameters-order.txt | 18 +- .../comprehensiveTest.txt | 18 +- 22 files changed, 415 insertions(+), 289 deletions(-) rename sqrl-planner/src/main/java/com/datasqrl/server/{SqrlPaginationUtil.java => OffsetPageInfoUtil.java} (93%) diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java index ab4e53a672..72c9f3e1d8 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java @@ -23,8 +23,8 @@ import com.datasqrl.server.ApiSource; import com.datasqrl.server.ApiSources; import com.datasqrl.server.GraphqlSchemaHandler; +import com.datasqrl.server.OffsetPageInfoUtil; import com.datasqrl.server.ScriptFiles; -import com.datasqrl.server.SqrlPaginationUtil; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -82,7 +82,7 @@ public LoadResult load(ServerPhysicalPlan serverPlan) { apiVersion -> new ApiSources( apiVersion.version(), - SqrlPaginationUtil.injectPaginationType(apiVersion.schema()), + OffsetPageInfoUtil.injectPaginationType(apiVersion.schema()), apiVersion.operations())) .toList(); injected.forEach(apiVersion -> graphqlSchemaHandler.validateSchema(apiVersion, serverPlan)); diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index 777f291f7b..aa10214f87 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -169,7 +169,9 @@ protected void visitQuery( .map(InputValueDefinition::getName) .anyMatch( name -> name.equals(SchemaConstants.LIMIT) || name.equals(SchemaConstants.OFFSET)); - var countSql = paged ? buildCountSql(tableFunction, executableJdbcReadQuery.getSql()) : null; + var countSql = paged ? buildCountSql(executableJdbcReadQuery.getSql()) : null; + var countWithEventTimesSql = + paged ? buildCountWithEventTimesSql(tableFunction, executableJdbcReadQuery.getSql()) : null; queryBase = new SqlQuery( executableJdbcReadQuery.getSql(), @@ -177,7 +179,8 @@ protected void visitQuery( hasLimitOrOffset ? PaginationType.LIMIT_AND_OFFSET : PaginationType.NONE, executableJdbcReadQuery.getCacheDuration().toMillis(), executableJdbcReadQuery.getDatabase(), - countSql); + countSql, + countWithEventTimesSql); var coordsBuilder = ArgumentLookupQueryCoords.builder() .parentType(parentType.getName()) @@ -189,24 +192,32 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } + /** Builds the companion COUNT(*) query for a paginated result. */ + private static String buildCountSql(String baseSql) { + return "SELECT COUNT(*) AS \"total_records\" FROM (" + baseSql + ") x"; + } + /** - * Builds the companion COUNT(*) query for a paginated result. Adds MIN/MAX over the designated - * rowtime column when the result has one, so {@code firstEventTime}/{@code lastEventTime} can be - * populated. The rowtime column name is the same identifier as in the base query's output. + * Variant of the count query that also computes MIN/MAX over the designated rowtime column for + * {@code firstEventTime}/{@code lastEventTime}. Returns null when the result has no rowtime. The + * rowtime column name is the same identifier as in the base query's output. */ - private static String buildCountSql(SqrlTableFunction tableFunction, String baseSql) { - var tsCol = - tableFunction.getRowTime().map(tableFunction::getField).map(RelDataTypeField::getName); - var select = new StringBuilder("SELECT COUNT(*) AS \"total_records\""); - tsCol.ifPresent( - col -> - select - .append(", MIN(\"") - .append(col) - .append("\") AS \"first_event_time\", MAX(\"") - .append(col) - .append("\") AS \"last_event_time\"")); - return select.append(" FROM (").append(baseSql).append(") x").toString(); + private static String buildCountWithEventTimesSql( + SqrlTableFunction tableFunction, String baseSql) { + return tableFunction + .getRowTime() + .map(tableFunction::getField) + .map(RelDataTypeField::getName) + .map( + col -> + "SELECT COUNT(*) AS \"total_records\", MIN(\"" + + col + + "\") AS \"first_event_time\", MAX(\"" + + col + + "\") AS \"last_event_time\" FROM (" + + baseSql + + ") x") + .orElse(null); } private static QueryParameterHandler convert(FunctionParameter fnParam) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java index 4d28afb5f9..a3c4c0d6e4 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java @@ -378,7 +378,7 @@ protected void visitQuery( boolean paged) { checkValidArrayNonNullType(atField.getType()); if (paged) { - SqrlPaginationUtil.validatePaginationType(registry); + OffsetPageInfoUtil.validatePaginationType(registry); var argNames = atField.getInputValueDefinitions().stream() .map(InputValueDefinition::getName) diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java index 632a086f80..22b812a493 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java @@ -141,10 +141,10 @@ private void walkTableFunction( typeDefinition.getName()); var resultType = (ObjectTypeDefinition) typeDefinition; - // A page wrapper ({results: [Element!], pagination: SqrlPagination}) is treated like a list of + // A page wrapper ({results: [Element!], pagination: OffsetPageInfo}) is treated like a list of // Element: validate/walk the element type against the function row type and compute pagination // metadata via a companion count query. - var pagedElement = SqrlPaginationUtil.getPagedElementType(resultType, registry); + var pagedElement = OffsetPageInfoUtil.getPagedElementType(resultType, registry); var paged = pagedElement.isPresent(); if (paged) { checkState( diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java similarity index 93% rename from sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java rename to sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index a88c5ab994..dee56b2b5c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/SqrlPaginationUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -31,15 +31,15 @@ /** * Opt-in pagination support: a query whose result type is a page wrapper ({@code {results: - * [Element!] pagination: SqrlPagination}}) returns its rows plus pagination metadata computed from + * [Element!] pagination: OffsetPageInfo}}) returns its rows plus pagination metadata computed from * a companion COUNT(*) query. This util detects the wrapper shape and injects/validates the - * standard {@code SqrlPagination} type. + * standard {@code OffsetPageInfo} type. */ -public final class SqrlPaginationUtil { +public final class OffsetPageInfoUtil { - private SqrlPaginationUtil() {} + private OffsetPageInfoUtil() {} - public static final String PAGINATION_TYPE_NAME = "SqrlPagination"; + public static final String PAGINATION_TYPE_NAME = "OffsetPageInfo"; /** Canonical field -> printed type, kept in declaration order for the injected SDL. */ private static final Map PAGINATION_FIELDS = new LinkedHashMap<>(); @@ -67,7 +67,7 @@ private static String buildCanonicalSdl() { } /** - * If the schema references {@code SqrlPagination} but does not define it, append the canonical + * If the schema references {@code OffsetPageInfo} but does not define it, append the canonical * definition (plus any missing scalar declarations). A user-provided definition is left untouched * here and validated later by {@link #validatePaginationType} within the schema validator's error * scope. Returns the (possibly rewritten) source. @@ -100,7 +100,7 @@ public static ApiSource injectPaginationType(ApiSource schema) { } /** - * Validates that a user-provided {@code SqrlPagination} type matches the canonical definition. + * Validates that a user-provided {@code OffsetPageInfo} type matches the canonical definition. * Must be called from within the schema validator so mismatches are reported like other schema * errors. No-op when the type is absent (it will have been injected) or unreferenced. */ @@ -119,7 +119,7 @@ public static void validatePaginationType(TypeDefinitionRegistry registry) { /** * Detects the page wrapper shape: an object type with exactly two fields, one a list of an object - * type (the results) and the other of type {@code SqrlPagination}. Returns the element object + * type (the results) and the other of type {@code OffsetPageInfo}. Returns the element object * type when the shape matches. */ public static Optional getPagedElementType( diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java index d1fcf58d5e..b0c86faf3e 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java @@ -301,18 +301,33 @@ public static class SqlQuery implements QueryBase { /** * Companion COUNT(*) query producing pagination metadata. When non-null, the query returns a - * page wrapper ({@code {results, pagination}}) rather than a bare list. + * page wrapper ({@code {results, pagination}}) rather than a bare list. Only executed when the + * request selects pagination fields that require it. */ @JsonInclude(JsonInclude.Include.NON_NULL) String countSql; + /** + * Variant of {@link #countSql} that additionally computes MIN/MAX over the rowtime column for + * {@code firstEventTime}/{@code lastEventTime}. Null when the result has no rowtime. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + String countWithEventTimesSql; + @Override public R accept(QueryBaseVisitor visitor, C context) { return visitor.visitSqlQuery(this, context); } public SqlQuery updateSql(String newSql) { - return new SqlQuery(newSql, parameters, pagination, cacheDurationMs, database, countSql); + return new SqlQuery( + newSql, + parameters, + pagination, + cacheDurationMs, + database, + countSql, + countWithEventTimesSql); } } @@ -495,13 +510,6 @@ public static class ResolvedSqlQuery implements ResolvedQuery { SqlQuery query; PreparedSqrlQuery preparedQueryContainer; - /** Prepared companion count query; null for non-paged queries or non-binding databases. */ - PreparedSqrlQuery preparedCountQueryContainer; - - public ResolvedSqlQuery(SqlQuery query, PreparedSqrlQuery preparedQueryContainer) { - this(query, preparedQueryContainer, null); - } - @Override public R accept(ResolvedQueryVisitor visitor, C context) { return visitor.visitResolvedSqlQuery(this, context); diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java index 26c7dc942f..03dd567292 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxJdbcClient.java @@ -44,13 +44,7 @@ public ResolvedQuery prepareQuery(SqlQuery query, ServerContext context) { var preparedQuery = sqlClient.preparedQuery(query.getSql()); - PreparedVertxSqrlQuery preparedCountQuery = null; - if (query.getCountSql() != null) { - preparedCountQuery = new PreparedVertxSqrlQuery(sqlClient.preparedQuery(query.getCountSql())); - } - - return new ResolvedSqlQuery( - query, new PreparedVertxSqrlQuery(preparedQuery), preparedCountQuery); + return new ResolvedSqlQuery(query, new PreparedVertxSqrlQuery(preparedQuery)); } @Override diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index 092434ccad..314a052c3b 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -77,17 +77,40 @@ private CompletableFuture runQueryInternal( ResolvedSqlQuery resolvedQuery, boolean isList, List paramObj) { var preparedQueryContainer = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedQueryContainer(); - var countContainer = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedCountQueryContainer(); var query = resolvedQuery.getQuery(); var unpreparedSqlQuery = query.getSql(); var database = query.getDatabase(); var paged = query.getCountSql() != null; - // The count query is bound with the base parameters only, without the runtime limit/offset. - var countParams = paged ? Tuple.from(List.copyOf(paramObj)) : null; + // Pagination metadata is computed lazily from the selection set: the aggregate query only + // runs when totals or event times are selected, and hasNextPage without totals is derived + // by fetching one extra row instead. + PageFieldNames fieldNames = null; + String aggregateSql = null; + var needNextWithoutAggregate = false; + if (paged) { + fieldNames = pageFieldNames(environment.getFieldType()); + var pag = fieldNames.paginationField(); + var selection = environment.getSelectionSet(); + var needEventTimes = selection.containsAnyOf(pag + "/firstEventTime", pag + "/lastEventTime"); + var needTotals = selection.containsAnyOf(pag + "/totalRecords", pag + "/totalPages"); + if (needEventTimes && query.getCountWithEventTimesSql() != null) { + aggregateSql = query.getCountWithEventTimesSql(); + } else if (needTotals) { + aggregateSql = query.getCountSql(); + } + needNextWithoutAggregate = + aggregateSql == null + && selection.containsAnyOf(pag + "/hasNextPage", pag + "/nextOffset"); + } + + // The aggregate query is bound with the base parameters only, without the runtime + // limit/offset. + var countParams = aggregateSql != null ? Tuple.from(List.copyOf(paramObj)) : null; int limitValue = Integer.MAX_VALUE; int offsetValue = 0; + var fetchExtraRow = false; switch (query.getPagination()) { case NONE: break; @@ -97,6 +120,11 @@ private CompletableFuture runQueryInternal( limitValue = limit.orElse(Integer.MAX_VALUE); offsetValue = offset.orElse(0); + // without a limit the page contains every remaining row, so there is no next page and + // no extra row to fetch + fetchExtraRow = needNextWithoutAggregate && limitValue != Integer.MAX_VALUE; + var fetchLimit = fetchExtraRow ? limitValue + 1 : limitValue; + // special case where database doesn't support binding for limit/offset => need // to execute dynamically if (!query.getDatabase().supportsLimitOffsetBinding) { @@ -104,11 +132,11 @@ private CompletableFuture runQueryInternal( unpreparedSqlQuery = AbstractQueryExecutionContext.addLimitOffsetToQuery( unpreparedSqlQuery, - limit.map(Object::toString).orElse("ALL"), - String.valueOf(offset.orElse(0))); + limit.isPresent() ? String.valueOf(fetchLimit) : "ALL", + String.valueOf(offsetValue)); } else { - paramObj.add(limit.orElse(Integer.MAX_VALUE)); - paramObj.add(offset.orElse(0)); + paramObj.add(fetchLimit); + paramObj.add(offsetValue); } break; default: @@ -127,18 +155,27 @@ private CompletableFuture runQueryInternal( } if (paged) { - Future> countFuture = - countContainer == null - ? serverContext.getSqlClient().execute(database, query.getCountSql(), countParams) - : serverContext.getSqlClient().execute(countContainer.preparedQuery(), countParams); + Future> aggregateFuture = + aggregateSql == null + ? Future.succeededFuture(null) + : serverContext.getSqlClient().execute(database, aggregateSql, countParams); var dataFuture = future; + var pageNames = fieldNames; var effectiveLimit = limitValue; var effectiveOffset = offsetValue; - Future.all(dataFuture, countFuture) + var extraRowFetched = fetchExtraRow; + var deriveNextFromResults = needNextWithoutAggregate; + Future.all(dataFuture, aggregateFuture) .map( c -> pagedResultMapper( - dataFuture.result(), countFuture.result(), effectiveLimit, effectiveOffset)) + dataFuture.result(), + aggregateFuture.result(), + pageNames, + effectiveLimit, + effectiveOffset, + extraRowFetched, + deriveNextFromResults)) .onSuccess(cf::complete) .onFailure( f -> { @@ -167,21 +204,40 @@ private Object resultMapper(RowSet r, boolean isList) { } private Object pagedResultMapper( - RowSet dataRows, RowSet countRows, int limit, int offset) { + RowSet dataRows, + RowSet aggregateRows, + PageFieldNames fieldNames, + int limit, + int offset, + boolean extraRowFetched, + boolean deriveNextFromResults) { var results = StreamSupport.stream(dataRows.spliterator(), false).map(Row::toJson).toList(); - var countIterator = countRows.iterator(); - var countJson = countIterator.hasNext() ? countIterator.next().toJson() : new JsonObject(); - var totalRecords = countJson.getLong("total_records", 0L); + Boolean hasNextPage = null; + if (extraRowFetched) { + hasNextPage = results.size() > limit; + if (hasNextPage) { + results = results.subList(0, limit); + } + } else if (deriveNextFromResults) { + hasNextPage = false; // no limit given: the page contains every remaining row + } + + Long totalRecords = null; + Object firstEventTime = null; + Object lastEventTime = null; + if (aggregateRows != null) { + var aggregateIterator = aggregateRows.iterator(); + var aggregateJson = + aggregateIterator.hasNext() ? aggregateIterator.next().toJson() : new JsonObject(); + totalRecords = aggregateJson.getLong("total_records", 0L); + firstEventTime = aggregateJson.getValue("first_event_time"); + lastEventTime = aggregateJson.getValue("last_event_time"); + } + var pagination = buildPaginationMetadata( - totalRecords, - limit, - offset, - countJson.getValue("first_event_time"), - countJson.getValue("last_event_time")); - - var fieldNames = pageFieldNames(environment.getFieldType()); + totalRecords, hasNextPage, limit, offset, firstEventTime, lastEventTime); return new JsonObject() .put(fieldNames.resultsField(), results) .put(fieldNames.paginationField(), pagination); @@ -211,23 +267,40 @@ private static PageFieldNames pageFieldNames(GraphQLOutputType fieldType) { private record PageFieldNames(String resultsField, String paginationField) {} + /** + * Builds the pagination metadata object. {@code totalRecords} and {@code hasNextPage} are null + * when the request did not select fields requiring them; the corresponding fields are then left + * out (GraphQL never reads unselected fields). + */ static JsonObject buildPaginationMetadata( - long totalRecords, int limit, int offset, Object firstEventTime, Object lastEventTime) { - int totalPages = limit == 0 ? 0 : (int) Math.ceil((double) totalRecords / limit); - int currentPage = limit == 0 ? 1 : offset / limit + 1; - boolean hasNextPage = (long) offset + limit < totalRecords; + Long totalRecords, + Boolean hasNextPage, + int limit, + int offset, + Object firstEventTime, + Object lastEventTime) { boolean hasPreviousPage = offset > 0; + var pagination = + new JsonObject() + .put("pageSize", limit) + .put("currentPage", limit == 0 ? 1 : offset / limit + 1) + .put("hasPreviousPage", hasPreviousPage) + .put( + "prevOffset", hasPreviousPage ? Integer.valueOf(Math.max(0, offset - limit)) : null) + .put("firstEventTime", firstEventTime) + .put("lastEventTime", lastEventTime); - return new JsonObject() - .put("totalRecords", totalRecords) - .put("pageSize", limit) - .put("currentPage", currentPage) - .put("totalPages", totalPages) - .put("hasNextPage", hasNextPage) - .put("hasPreviousPage", hasPreviousPage) - .put("nextOffset", hasNextPage ? Integer.valueOf(offset + limit) : null) - .put("prevOffset", hasPreviousPage ? Integer.valueOf(Math.max(0, offset - limit)) : null) - .put("firstEventTime", firstEventTime) - .put("lastEventTime", lastEventTime); + if (totalRecords != null) { + pagination + .put("totalRecords", totalRecords) + .put("totalPages", limit == 0 ? 0 : (int) Math.ceil((double) totalRecords / limit)); + hasNextPage = (long) offset + limit < totalRecords; + } + if (hasNextPage != null) { + pagination + .put("hasNextPage", hasNextPage) + .put("nextOffset", hasNextPage ? Integer.valueOf(offset + limit) : null); + } + return pagination; } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java index 63d24ce7b1..6ffd145af3 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java @@ -172,6 +172,7 @@ private RootGraphQLModel getCustomerModel() { PaginationType.NONE, 0, DatabaseType.POSTGRES, + null, null)) .build()) .build()) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java index dec6f9ed46..699e9993cf 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java @@ -23,7 +23,7 @@ class PaginationMetadataTest { @Test void givenEmptyResult_whenBuildMetadata_thenSinglePageNoRecords() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(0, 10, 0, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(0L, null, 10, 0, null, null); assertThat(json.getLong("totalRecords")).isZero(); assertThat(json.getInteger("pageSize")).isEqualTo(10); @@ -37,7 +37,7 @@ void givenEmptyResult_whenBuildMetadata_thenSinglePageNoRecords() { @Test void givenFirstPage_whenBuildMetadata_thenHasNextNoPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 0, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 0, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(1); assertThat(json.getInteger("totalPages")).isEqualTo(3); @@ -49,7 +49,7 @@ void givenFirstPage_whenBuildMetadata_thenHasNextNoPrevious() { @Test void givenMiddlePage_whenBuildMetadata_thenHasBothNeighbours() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 10, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 10, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(2); assertThat(json.getBoolean("hasNextPage")).isTrue(); @@ -60,7 +60,7 @@ void givenMiddlePage_whenBuildMetadata_thenHasBothNeighbours() { @Test void givenLastPartialPage_whenBuildMetadata_thenNoNextHasPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 20, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 20, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(3); assertThat(json.getBoolean("hasNextPage")).isFalse(); @@ -71,7 +71,7 @@ void givenLastPartialPage_whenBuildMetadata_thenNoNextHasPrevious() { @Test void givenOffsetBeyondTotal_whenBuildMetadata_thenNoNextPage() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 10, 30, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 30, null, null); assertThat(json.getBoolean("hasNextPage")).isFalse(); assertThat(json.getBoolean("hasPreviousPage")).isTrue(); @@ -80,7 +80,7 @@ void givenOffsetBeyondTotal_whenBuildMetadata_thenNoNextPage() { @Test void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25, 0, 0, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 0, 0, null, null); assertThat(json.getInteger("totalPages")).isZero(); assertThat(json.getInteger("currentPage")).isEqualTo(1); @@ -90,9 +90,39 @@ void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { void givenEventTimes_whenBuildMetadata_thenPassedThrough() { var json = VertxQueryExecutionContext.buildPaginationMetadata( - 5, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); + 5L, null, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); assertThat(json.getString("firstEventTime")).isEqualTo("2024-01-01T00:00:00Z"); assertThat(json.getString("lastEventTime")).isEqualTo("2024-01-02T00:00:00Z"); } + + @Test + void givenNoTotalsQueried_whenBuildMetadata_thenTotalsOmitted() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(null, true, 10, 10, null, null); + + assertThat(json.containsKey("totalRecords")).isFalse(); + assertThat(json.containsKey("totalPages")).isFalse(); + assertThat(json.getBoolean("hasNextPage")).isTrue(); + assertThat(json.getInteger("nextOffset")).isEqualTo(20); + assertThat(json.getBoolean("hasPreviousPage")).isTrue(); + assertThat(json.getInteger("prevOffset")).isZero(); + } + + @Test + void givenNoNextPageInfoQueried_whenBuildMetadata_thenNextFieldsOmitted() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(null, null, 10, 0, null, null); + + assertThat(json.containsKey("hasNextPage")).isFalse(); + assertThat(json.containsKey("nextOffset")).isFalse(); + assertThat(json.getInteger("pageSize")).isEqualTo(10); + assertThat(json.getBoolean("hasPreviousPage")).isFalse(); + } + + @Test + void givenTotalsQueried_whenBuildMetadata_thenHasNextDerivedFromTotals() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, false, 10, 0, null, null); + + assertThat(json.getBoolean("hasNextPage")).isTrue(); + assertThat(json.getInteger("nextOffset")).isEqualTo(10); + } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls index 678970c578..e9b95869e3 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls @@ -13,7 +13,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls index edf18ab299..8e8772da3f 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-subscription.graphqls @@ -13,7 +13,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls index 3b7b998789..1c48801473 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls @@ -14,7 +14,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls index 1dd177af24..bd7e007f4b 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-wrong-pagination-type.graphqls @@ -3,7 +3,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -18,7 +18,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls index 9304f1b873..fce97c9ce1 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls @@ -13,7 +13,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls index 9809dd9dad..c80fc870f8 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls @@ -3,7 +3,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -27,12 +27,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt index 6908b873db..6d44cb60fb 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt @@ -19,8 +19,8 @@ in script:comprehensiveTest.sqrl [51:1]: CustomerTimeWindow := SELECT ^ -[FATAL] User-defined SqrlPagination does not match the expected definition: -type SqrlPagination { +[FATAL] User-defined OffsetPageInfo does not match the expected definition: +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -36,6 +36,6 @@ type SqrlPagination { in script:comprehensiveTest-fail-paged-wrong-pagination-type.graphqls [6:1]: scalar Long -type SqrlPagination { +type OffsetPageInfo { ^ diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt index 606e42994f..084566fe4d 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt @@ -285,7 +285,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -308,7 +308,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -336,7 +336,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -349,7 +349,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -364,7 +364,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -535,7 +535,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -548,7 +548,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -572,12 +572,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index bf34820de8..436f8e9224 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -285,7 +285,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -308,7 +308,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -336,7 +336,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -349,7 +349,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -364,7 +364,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -535,7 +535,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -548,7 +548,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -572,12 +572,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { @@ -1617,73 +1617,74 @@ WITH ( EXECUTE STATEMENT SET BEGIN INSERT INTO `default_catalog`.`default_database`.`AnotherCustomer_1` SELECT * - FROM `default_catalog`.`default_database`.`AnotherCustomer` +FROM `default_catalog`.`default_database`.`AnotherCustomer` ; INSERT INTO `default_catalog`.`default_database`.`Customer_2` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`Customer_3` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerSubscription` - ; - INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerTimeWindow` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerTimeWindow` - ; - INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` - SELECT * - FROM `default_catalog`.`default_database`.`ExplicitDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` - SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` - FROM `default_catalog`.`default_database`.`ExternalOrders` - ; - INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` - SELECT * - FROM `default_catalog`.`default_database`.`InvalidDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`Orders_13` - SELECT * - FROM `default_catalog`.`default_database`.`Orders` - ; - INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` - SELECT * - FROM `default_catalog`.`default_database`.`SelectCustomers` - ; - INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` - SELECT * - FROM `default_catalog`.`default_database`.`TemporalJoin` - ; - INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` - SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` - FROM `default_catalog`.`default_database`.`TemporalJoin` - ; - INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` - SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` - FROM `default_catalog`.`default_database`.`UnnestOrders` - ; - END +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`Customer_3` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerSubscription` +; +INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerTimeWindow` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerTimeWindow` +; +INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` +SELECT * +FROM `default_catalog`.`default_database`.`ExplicitDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` +SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` +FROM `default_catalog`.`default_database`.`ExternalOrders` +; +INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` +SELECT * +FROM `default_catalog`.`default_database`.`InvalidDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`Orders_13` +SELECT * +FROM `default_catalog`.`default_database`.`Orders` +; +INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` +SELECT * +FROM `default_catalog`.`default_database`.`SelectCustomers` +; +INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` +SELECT * +FROM `default_catalog`.`default_database`.`TemporalJoin` +; +INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` +SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` +FROM `default_catalog`.`default_database`.`TemporalJoin` +; +INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` +SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` +FROM `default_catalog`.`default_database`.`UnnestOrders` +; +END; + >>>kafka.json { "topics" : [ @@ -1743,11 +1744,13 @@ CREATE TABLE IF NOT EXISTS "SelectCustomers" ("customerid" BIGINT NOT NULL, "ema CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, "customerid0" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("id","time")); CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); -CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name") +CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); + >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * FROM "ExternalOrders" AS "ExternalOrders0" - INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" + INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid"; + >>>vertx.json { "models" : { @@ -1775,7 +1778,8 @@ FROM "ExternalOrders" AS "ExternalOrders0" "pagination" : "LIMIT_AND_OFFSET", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"CustomerByTime2\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" } } } @@ -1812,7 +1816,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: SqrlPagination\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n\ntype SqrlPagination {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 2106f83733..39a8ec0209 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -285,7 +285,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -308,7 +308,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -336,7 +336,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -349,7 +349,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -364,7 +364,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -535,7 +535,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -548,7 +548,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -572,12 +572,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { @@ -1617,73 +1617,74 @@ WITH ( EXECUTE STATEMENT SET BEGIN INSERT INTO `default_catalog`.`default_database`.`AnotherCustomer_1` SELECT * - FROM `default_catalog`.`default_database`.`AnotherCustomer` +FROM `default_catalog`.`default_database`.`AnotherCustomer` ; INSERT INTO `default_catalog`.`default_database`.`Customer_2` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`Customer_3` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` - SELECT * - FROM `default_catalog`.`default_database`.`Customer` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerSubscription` - ; - INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerTimeWindow` - ; - INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` - SELECT * - FROM `default_catalog`.`default_database`.`CustomerTimeWindow` - ; - INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` - SELECT * - FROM `default_catalog`.`default_database`.`ExplicitDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` - SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` - FROM `default_catalog`.`default_database`.`ExternalOrders` - ; - INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` - SELECT * - FROM `default_catalog`.`default_database`.`InvalidDistinct` - ; - INSERT INTO `default_catalog`.`default_database`.`Orders_13` - SELECT * - FROM `default_catalog`.`default_database`.`Orders` - ; - INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` - SELECT * - FROM `default_catalog`.`default_database`.`SelectCustomers` - ; - INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` - SELECT * - FROM `default_catalog`.`default_database`.`TemporalJoin` - ; - INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` - SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` - FROM `default_catalog`.`default_database`.`TemporalJoin` - ; - INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` - SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` - FROM `default_catalog`.`default_database`.`UnnestOrders` - ; - END +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`Customer_3` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerByMultipleTime_4` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerByTime2_5` +SELECT * +FROM `default_catalog`.`default_database`.`Customer` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerFilteredDistinct_6` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerFilteredDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerSubscription_7` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerSubscription` +; +INSERT INTO `default_catalog`.`default_database`.`TimeWindow_8` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerTimeWindow` +; +INSERT INTO `default_catalog`.`default_database`.`CustomerTimeWindow_9` +SELECT * +FROM `default_catalog`.`default_database`.`CustomerTimeWindow` +; +INSERT INTO `default_catalog`.`default_database`.`ExplicitDistinct_10` +SELECT * +FROM `default_catalog`.`default_database`.`ExplicitDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`ExternalOrders_11` +SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries` +FROM `default_catalog`.`default_database`.`ExternalOrders` +; +INSERT INTO `default_catalog`.`default_database`.`InvalidDistinct_12` +SELECT * +FROM `default_catalog`.`default_database`.`InvalidDistinct` +; +INSERT INTO `default_catalog`.`default_database`.`Orders_13` +SELECT * +FROM `default_catalog`.`default_database`.`Orders` +; +INSERT INTO `default_catalog`.`default_database`.`SelectCustomers_14` +SELECT * +FROM `default_catalog`.`default_database`.`SelectCustomers` +; +INSERT INTO `default_catalog`.`default_database`.`MyPrintSink_ex1` +SELECT * +FROM `default_catalog`.`default_database`.`TemporalJoin` +; +INSERT INTO `default_catalog`.`default_database`.`TemporalJoin_15` +SELECT `id`, `customerid`, `time`, `to_jsonb`(`entries`) AS `entries`, `customerid0`, CAST(`timestamp` AS TIMESTAMP(3) WITH LOCAL TIME ZONE) AS `timestamp`, `name` +FROM `default_catalog`.`default_database`.`TemporalJoin` +; +INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` +SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` +FROM `default_catalog`.`default_database`.`UnnestOrders` +; +END; + >>>kafka.json { "topics" : [ @@ -1743,11 +1744,13 @@ CREATE TABLE IF NOT EXISTS "SelectCustomers" ("customerid" BIGINT NOT NULL, "ema CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "entries" JSONB, "customerid0" BIGINT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "name" TEXT NOT NULL, PRIMARY KEY ("id","time")); CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); -CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name") +CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); + >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * FROM "ExternalOrders" AS "ExternalOrders0" - INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" + INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid"; + >>>vertx.json { "models" : { @@ -1794,7 +1797,8 @@ FROM "ExternalOrders" AS "ExternalOrders0" "pagination" : "LIMIT_AND_OFFSET", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" } } }, @@ -1825,7 +1829,8 @@ FROM "ExternalOrders" AS "ExternalOrders0" "pagination" : "LIMIT_AND_OFFSET", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" } } } @@ -1835,7 +1840,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" "operations" : [ ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype SqrlPagination {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: SqrlPagination\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: SqrlPagination\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt index dd2677f3c9..d2fd49c7b0 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt @@ -285,7 +285,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -308,7 +308,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -336,7 +336,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -349,7 +349,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -364,7 +364,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -535,7 +535,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -548,7 +548,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -572,12 +572,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt index 413217a0d9..8d8fec4e4c 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt @@ -285,7 +285,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -308,7 +308,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -336,7 +336,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -349,7 +349,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! } @@ -364,7 +364,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -535,7 +535,7 @@ type Customer { type CustomerPage { results: [Customer!] - pagination: SqrlPagination + pagination: OffsetPageInfo } type Query { @@ -548,7 +548,7 @@ scalar DateTime "A 64-bit signed integer" scalar Long -type SqrlPagination { +type OffsetPageInfo { totalRecords: Long! pageSize: Int! currentPage: Int! @@ -572,12 +572,12 @@ type Customer { type CustomerRelatedPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type CustomerPage { items: [Customer!] - meta: SqrlPagination + meta: OffsetPageInfo } type Query { From d61155967629678c56e90d97a13c3c6a5fd8cd3d Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 8 Jul 2026 12:11:18 -0300 Subject: [PATCH 03/16] test: Prove lazy pagination metadata via recorded SQL in PagedQueryIT Signed-off-by: Marvin Froeder --- .../com/datasqrl/server/PagedQueryIT.java | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java new file mode 100644 index 0000000000..42f58b5d8a --- /dev/null +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -0,0 +1,367 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.server; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datasqrl.server.graphql.CustomScalars; +import com.datasqrl.server.graphql.GraphQLEngineBuilder; +import com.datasqrl.server.graphql.RootGraphQLModel; +import com.datasqrl.server.graphql.RootGraphQLModel.ArgumentLookupQueryCoords; +import com.datasqrl.server.graphql.RootGraphQLModel.QueryWithArguments; +import com.datasqrl.server.graphql.RootGraphQLModel.SqlQuery; +import com.datasqrl.server.graphql.RootGraphQLModel.StringSchema; +import com.datasqrl.server.jdbc.DatabaseType; +import com.datasqrl.server.jdbc.VertxJdbcClient; +import com.datasqrl.server.jdbc.VertxParamArgumentTypeMapper; +import graphql.ExecutionInput; +import graphql.GraphQL; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import io.vertx.junit5.VertxExtension; +import io.vertx.pgclient.PgBuilder; +import io.vertx.pgclient.PgConnectOptions; +import io.vertx.sqlclient.PoolOptions; +import io.vertx.sqlclient.PrepareOptions; +import io.vertx.sqlclient.PreparedQuery; +import io.vertx.sqlclient.Query; +import io.vertx.sqlclient.Row; +import io.vertx.sqlclient.RowSet; +import io.vertx.sqlclient.SqlClient; +import io.vertx.sqlclient.SqlResult; +import io.vertx.sqlclient.Tuple; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; +import java.util.stream.Collector; +import lombok.SneakyThrows; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; + +/** + * Proves that pagination metadata is computed lazily from the selection set: the aggregate query + * only runs when totals or event times are selected, event times pick the MIN/MAX variant, and + * {@code hasNextPage} alone is answered by fetching LIMIT+1 rows instead of any count query. + */ +@ExtendWith(VertxExtension.class) +@Testcontainers +class PagedQueryIT { + + private static final String BASE_SQL = "SELECT customerid, ts FROM customer ORDER BY customerid"; + private static final String COUNT_SQL = + "SELECT COUNT(*) AS \"total_records\" FROM (" + BASE_SQL + ") x"; + private static final String COUNT_WITH_EVENT_TIMES_SQL = + "SELECT COUNT(*) AS \"total_records\", MIN(\"ts\") AS \"first_event_time\"," + + " MAX(\"ts\") AS \"last_event_time\" FROM (" + + BASE_SQL + + ") x"; + + @Container + private static final PostgreSQLContainer postgresContainer = + new PostgreSQLContainer(DockerImageName.parse("postgres:16")) + .withDatabaseName("datasqrl") + .withUsername("foo") + .withPassword("secret"); + + private SqlClient client; + private RecordingSqlClient recordingClient; + private GraphQL graphQL; + + @BeforeEach + void init(Vertx vertx) { + var options = new PgConnectOptions(); + options.setDatabase(postgresContainer.getDatabaseName()); + options.setHost(postgresContainer.getHost()); + options.setPort(postgresContainer.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT)); + options.setUser(postgresContainer.getUsername()); + options.setPassword(postgresContainer.getPassword()); + + client = PgBuilder.client().with(new PoolOptions()).connectingTo(options).using(vertx).build(); + await(client.query("DROP TABLE IF EXISTS customer").execute()); + await(client.query("CREATE TABLE customer (customerid INT, ts TIMESTAMPTZ)").execute()); + await( + client + .query( + """ + INSERT INTO customer VALUES + (1, '2024-01-01T00:00:00Z'), + (2, '2024-01-02T00:00:00Z'), + (3, '2024-01-03T00:00:00Z'), + (4, '2024-01-04T00:00:00Z'), + (5, '2024-01-05T00:00:00Z') + """) + .execute()); + + recordingClient = new RecordingSqlClient(client); + graphQL = + getPagedModel() + .accept( + new GraphQLEngineBuilder.Builder() + .withExtendedScalarTypes(CustomScalars.getExtendedScalars()) + .build(), + new VertxServerContext( + new VertxJdbcClient(Map.of(DatabaseType.POSTGRES, recordingClient)), + null, + null, + new VertxParamArgumentTypeMapper())) + .build(); + recordingClient.executed.clear(); + } + + @AfterEach + void after() { + client.close(); + } + + @Test + void givenOnlyResultsSelected_whenQuery_thenNoAggregateQueryRuns() { + var customers = execute("{ customers(limit: 2, offset: 0) { results { customerid } } }"); + + assertThat(results(customers)).extracting("customerid").containsExactly(1, 2); + assertThat(recordingClient.executed).hasSize(1); + assertThat(recordingClient.executed.get(0).params().getInteger(0)).isEqualTo(2); + } + + @Test + void givenOnlyOffsetDerivedFieldsSelected_whenQuery_thenNoAggregateQueryRuns() { + var customers = + execute( + "{ customers(limit: 2, offset: 2) { results { customerid }" + + " pagination { pageSize currentPage hasPreviousPage prevOffset } } }"); + + assertThat(recordingClient.executed).hasSize(1); + var pagination = pagination(customers); + assertThat(pagination) + .containsEntry("pageSize", 2) + .containsEntry("currentPage", 2) + .containsEntry("hasPreviousPage", true) + .containsEntry("prevOffset", 0); + } + + @Test + void givenOnlyHasNextPageSelected_whenQuery_thenLimitPlusOneReplacesCount() { + var customers = + execute( + "{ customers(limit: 2, offset: 0) { results { customerid }" + + " pagination { hasNextPage nextOffset } } }"); + + assertThat(recordingClient.executed).hasSize(1); + // the single data query fetched limit+1 rows and the extra row was trimmed + assertThat(recordingClient.executed.get(0).params().getInteger(0)).isEqualTo(3); + assertThat(results(customers)).extracting("customerid").containsExactly(1, 2); + assertThat(pagination(customers)) + .containsEntry("hasNextPage", true) + .containsEntry("nextOffset", 2); + } + + @Test + void givenHasNextPageSelectedOnLastPage_whenQuery_thenNoNextPage() { + var customers = + execute( + "{ customers(limit: 2, offset: 4) { results { customerid }" + + " pagination { hasNextPage nextOffset } } }"); + + assertThat(recordingClient.executed).hasSize(1); + assertThat(results(customers)).extracting("customerid").containsExactly(5); + assertThat(pagination(customers)).containsEntry("hasNextPage", false); + } + + @Test + void givenTotalsSelected_whenQuery_thenPlainCountRunsWithoutExtraRow() { + var customers = + execute( + "{ customers(limit: 2, offset: 0) { results { customerid }" + + " pagination { totalRecords totalPages hasNextPage } } }"); + + assertThat(recordingClient.executed).hasSize(2); + assertThat(sqlOf(recordingClient.executed)).contains(COUNT_SQL); + // hasNextPage is derived from the count, so the data query does not fetch an extra row + var dataStatement = + recordingClient.executed.stream().filter(s -> !s.sql().contains("COUNT")).findFirst(); + assertThat(dataStatement).isPresent(); + assertThat(dataStatement.get().params().getInteger(0)).isEqualTo(2); + assertThat(pagination(customers)) + .containsEntry("totalRecords", 5L) + .containsEntry("totalPages", 3) + .containsEntry("hasNextPage", true); + } + + @Test + void givenEventTimesSelected_whenQuery_thenMinMaxVariantRuns() { + var customers = + execute( + "{ customers(limit: 2, offset: 2) { results { customerid }" + + " pagination { totalRecords firstEventTime lastEventTime } } }"); + + assertThat(recordingClient.executed).hasSize(2); + assertThat(sqlOf(recordingClient.executed)).contains(COUNT_WITH_EVENT_TIMES_SQL); + var pagination = pagination(customers); + assertThat(pagination).containsEntry("totalRecords", 5L); + // MIN/MAX cover the whole result, not just the requested page + assertThat(String.valueOf(pagination.get("firstEventTime"))).startsWith("2024-01-01"); + assertThat(String.valueOf(pagination.get("lastEventTime"))).startsWith("2024-01-05"); + } + + @SneakyThrows + private Map execute(String query) { + var result = graphQL.execute(ExecutionInput.newExecutionInput().query(query).build()); + assertThat(result.getErrors()).isEmpty(); + Map data = result.getData(); + return (Map) data.get("customers"); + } + + @SuppressWarnings("unchecked") + private static List> results(Map customers) { + return (List>) customers.get("results"); + } + + @SuppressWarnings("unchecked") + private static Map pagination(Map customers) { + return (Map) customers.get("pagination"); + } + + private static List sqlOf(List statements) { + return statements.stream().map(ExecutedStatement::sql).toList(); + } + + private RootGraphQLModel getPagedModel() { + return RootGraphQLModel.builder() + .schema( + StringSchema.builder() + .schema( + """ + scalar DateTime + scalar Long + type Query { + customers(limit: Int = 10, offset: Int = 0): CustomerPage! + } + type Customer { + customerid: Int + } + type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo + } + type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime + } + """) + .build()) + .query( + ArgumentLookupQueryCoords.builder() + .parentType("Query") + .fieldName("customers") + .exec( + QueryWithArguments.builder() + .query( + new SqlQuery( + BASE_SQL, + List.of(), + PaginationType.LIMIT_AND_OFFSET, + 0, + DatabaseType.POSTGRES, + COUNT_SQL, + COUNT_WITH_EVENT_TIMES_SQL)) + .build()) + .build()) + .build(); + } + + @SneakyThrows + private static T await(Future future) { + return future.toCompletionStage().toCompletableFuture().get(); + } + + private record ExecutedStatement(String sql, Tuple params) {} + + /** Delegating {@link SqlClient} that records every executed statement with its bound tuple. */ + private static final class RecordingSqlClient implements SqlClient { + + private final SqlClient delegate; + final List executed = new CopyOnWriteArrayList<>(); + + RecordingSqlClient(SqlClient delegate) { + this.delegate = delegate; + } + + @Override + public Query> query(String sql) { + return delegate.query(sql); + } + + @Override + public PreparedQuery> preparedQuery(String sql) { + return recording(sql, delegate.preparedQuery(sql)); + } + + @Override + public PreparedQuery> preparedQuery(String sql, PrepareOptions options) { + return recording(sql, delegate.preparedQuery(sql, options)); + } + + @Override + public Future close() { + return delegate.close(); + } + + private PreparedQuery> recording(String sql, PreparedQuery> delegate) { + return new PreparedQuery<>() { + @Override + public Future> execute(Tuple tuple) { + executed.add(new ExecutedStatement(sql, tuple)); + return delegate.execute(tuple); + } + + @Override + public Future> execute() { + executed.add(new ExecutedStatement(sql, Tuple.tuple())); + return delegate.execute(); + } + + @Override + public Future> executeBatch(List batch) { + throw new UnsupportedOperationException(); + } + + @Override + public PreparedQuery> collecting(Collector collector) { + throw new UnsupportedOperationException(); + } + + @Override + public PreparedQuery> mapping(Function mapper) { + throw new UnsupportedOperationException(); + } + }; + } + } +} From 5720bc5605c2b3fdfa55a17e15e498b100a21bad Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 8 Jul 2026 12:53:03 -0300 Subject: [PATCH 04/16] feat: Add compiler config to generate paginated results in inferred GraphQL schema Signed-off-by: Marvin Froeder --- .../config/CompilerApiConfigImpl.java | 5 + .../java/com/datasqrl/config/PackageJson.java | 2 + .../datasqrl/server/GraphqlSchemaFactory.java | 55 +- .../datasqrl/server/OffsetPageInfoUtil.java | 27 + .../src/main/resources/default-package.json | 3 +- .../resources/jsonSchema/packageSchema.json | 3 + .../clickstream-package-paginated.txt | 217 +++++++ .../clickstream-package-paginated.txt | 535 ++++++++++++++++++ .../clickstream/package-paginated.json | 17 + 9 files changed, 856 insertions(+), 8 deletions(-) create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/usecases/clickstream/package-paginated.json diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java b/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java index 367aeed8a3..1e3a38cb6c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java @@ -57,6 +57,11 @@ public int getDefaultLimit() { return sqrlConfig.asInt("default-limit").get(); } + @Override + public boolean generatePaginatedResults() { + return sqrlConfig.asBool("paginated-results").withDefault(false).get(); + } + public enum Endpoints { OPS_ONLY, // only support the pre-defined operations in the GraphQL API, do not support flexible // GraphQL queries TODO: not yet implemented diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/PackageJson.java b/sqrl-planner/src/main/java/com/datasqrl/config/PackageJson.java index 680dcddd5c..ac989c0bb0 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/PackageJson.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/PackageJson.java @@ -79,6 +79,8 @@ interface CompilerApiConfig { int getMaxResultDepth(); int getDefaultLimit(); + + boolean generatePaginatedResults(); } interface ExplainConfig { diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaFactory.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaFactory.java index cfe46976b2..e35fdfdd8b 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaFactory.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaFactory.java @@ -43,6 +43,7 @@ import graphql.schema.GraphQLDirective; import graphql.schema.GraphQLEnumType; import graphql.schema.GraphQLFieldDefinition; +import graphql.schema.GraphQLNonNull; import graphql.schema.GraphQLObjectType; import graphql.schema.GraphQLOutputType; import graphql.schema.GraphQLSchema; @@ -72,11 +73,13 @@ public class GraphqlSchemaFactory { private final boolean extendedScalarTypes; private final boolean addApiDirective; private final int defaultLimit; + private final boolean generatePaginatedResults; public GraphqlSchemaFactory(CompilerConfig config) { this.extendedScalarTypes = config.isExtendedScalarTypes(); this.addApiDirective = !config.getApiConfig().isGraphQLProtocolOnly(); this.defaultLimit = config.getApiConfig().getDefaultLimit(); + this.generatePaginatedResults = config.getApiConfig().generatePaginatedResults(); } public GraphQLSchema generate(ServerPhysicalPlan serverPlan) { @@ -284,9 +287,7 @@ private Optional createRootType( final var type = tableFunctionsType == AccessModifier.QUERY - ? (GraphQLOutputType) - wrapMultiplicity( - createTypeReference(tableFunction), tableFunction.getMultiplicity()) + ? createQueryResultType(tableFunction) : createTypeReference( tableFunction); // type is nullable because there can be no update in the // subscription @@ -346,13 +347,15 @@ private Optional createRelationshipField(SqrlTableFuncti } // reference the type that will be defined when the table function relationship is processed + var fieldType = + relationship.getVisibility().access() == AccessModifier.QUERY + ? createQueryResultType(relationship) + : (GraphQLOutputType) + wrapMultiplicity(createTypeReference(relationship), relationship.getMultiplicity()); var field = GraphQLFieldDefinition.newFieldDefinition() .name(fieldName) - .type( - (GraphQLOutputType) - wrapMultiplicity( - createTypeReference(relationship), relationship.getMultiplicity())) + .type(fieldType) .arguments(createArguments(relationship)); relationship.getDocumentation().getDocStringOpt().ifPresent(field::description); @@ -432,6 +435,44 @@ private GraphQLOutputType createTypeReference(SqrlTableFunction tableFunction) { return new GraphQLTypeReference(typeName); } + /** + * Result type of a query table function: when paginated results are enabled, MANY results are + * wrapped in a {@code Page {results, pagination}} type so the server computes {@code + * OffsetPageInfo} metadata; otherwise the plain multiplicity-wrapped type. + */ + private GraphQLOutputType createQueryResultType(SqrlTableFunction tableFunction) { + if (generatePaginatedResults && tableFunction.getMultiplicity() == Multiplicity.MANY) { + return GraphQLNonNull.nonNull(createPageType(tableFunction)); + } + return (GraphQLOutputType) + wrapMultiplicity(createTypeReference(tableFunction), tableFunction.getMultiplicity()); + } + + private GraphQLTypeReference createPageType(SqrlTableFunction tableFunction) { + var elementType = (GraphQLTypeReference) createTypeReference(tableFunction); + var pageTypeName = elementType.getName() + "Page"; + if (!definedTypeNames.contains(pageTypeName)) { + definedTypeNames.add(pageTypeName); + objectTypes.add( + GraphQLObjectType.newObject() + .name(pageTypeName) + .field( + GraphQLFieldDefinition.newFieldDefinition() + .name("results") + .type((GraphQLOutputType) wrapMultiplicity(elementType, Multiplicity.MANY))) + .field( + GraphQLFieldDefinition.newFieldDefinition() + .name("pagination") + .type(new GraphQLTypeReference(OffsetPageInfoUtil.PAGINATION_TYPE_NAME))) + .build()); + if (!definedTypeNames.contains(OffsetPageInfoUtil.PAGINATION_TYPE_NAME)) { + definedTypeNames.add(OffsetPageInfoUtil.PAGINATION_TYPE_NAME); + objectTypes.add(OffsetPageInfoUtil.createPageInfoType()); + } + } + return new GraphQLTypeReference(pageTypeName); + } + public static final String API_DIRECTIVE_NAME = "api"; private void addApiDirectiveTypes(GraphQLSchema.Builder graphQLSchemaBuilder) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index dee56b2b5c..8c19674d9d 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -17,12 +17,18 @@ import static com.datasqrl.server.util.GraphqlCheckUtil.checkState; +import com.datasqrl.server.graphql.CustomScalars; +import graphql.Scalars; import graphql.language.FieldDefinition; import graphql.language.ListType; import graphql.language.NonNullType; import graphql.language.ObjectTypeDefinition; import graphql.language.Type; import graphql.language.TypeName; +import graphql.schema.GraphQLFieldDefinition; +import graphql.schema.GraphQLNonNull; +import graphql.schema.GraphQLObjectType; +import graphql.schema.GraphQLOutputType; import graphql.schema.idl.SchemaParser; import graphql.schema.idl.TypeDefinitionRegistry; import java.util.LinkedHashMap; @@ -59,6 +65,27 @@ private OffsetPageInfoUtil() {} private static final String CANONICAL_SDL = buildCanonicalSdl(); + /** Printed type -> GraphQL type, used to build the schema type from the canonical fields. */ + private static final Map GRAPHQL_TYPES = + Map.of( + "Long!", GraphQLNonNull.nonNull(CustomScalars.LONG), + "Int!", GraphQLNonNull.nonNull(Scalars.GraphQLInt), + "Boolean!", GraphQLNonNull.nonNull(Scalars.GraphQLBoolean), + "Int", Scalars.GraphQLInt, + "DateTime", CustomScalars.FLEXIBLE_DATETIME); + + /** Builds the canonical {@code OffsetPageInfo} object type for generated schemas. */ + public static GraphQLObjectType createPageInfoType() { + var builder = GraphQLObjectType.newObject().name(PAGINATION_TYPE_NAME); + PAGINATION_FIELDS.forEach( + (name, type) -> + builder.field( + GraphQLFieldDefinition.newFieldDefinition() + .name(name) + .type(GRAPHQL_TYPES.get(type)))); + return builder.build(); + } + private static String buildCanonicalSdl() { var sb = new StringBuilder("type ").append(PAGINATION_TYPE_NAME).append(" {\n"); PAGINATION_FIELDS.forEach( diff --git a/sqrl-planner/src/main/resources/default-package.json b/sqrl-planner/src/main/resources/default-package.json index 38ab8fb990..a50a89217a 100644 --- a/sqrl-planner/src/main/resources/default-package.json +++ b/sqrl-planner/src/main/resources/default-package.json @@ -18,7 +18,8 @@ "endpoints": "FULL", "add-prefix": true, "max-result-depth": 3, - "default-limit": 10 + "default-limit": 10, + "paginated-results": false } }, "engines": { diff --git a/sqrl-planner/src/main/resources/jsonSchema/packageSchema.json b/sqrl-planner/src/main/resources/jsonSchema/packageSchema.json index b3bccfd9ca..aee0e075be 100644 --- a/sqrl-planner/src/main/resources/jsonSchema/packageSchema.json +++ b/sqrl-planner/src/main/resources/jsonSchema/packageSchema.json @@ -97,6 +97,9 @@ }, "default-limit": { "type": "integer" + }, + "paginated-results": { + "type": "boolean" } }, "additionalProperties": false diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt new file mode 100644 index 0000000000..22e0f47677 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt @@ -0,0 +1,217 @@ +>>>pipeline_explain.json +[ { + "id" : "access:Click", + "name" : "Click", + "type" : "query", + "stage" : "postgres", + "inputs" : [ "default_catalog.default_database.Click" ], + "annotations" : [ { + "name" : "stream-root", + "description" : "Click" + }, { + "name" : "base-table", + "description" : "Click" + } ], + "plan" : "LogicalProject(url=[$0], timestamp=[$1], userid=[$2])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "sql" : "SELECT *\nFROM `default_catalog`.`default_database`.`Click`" +}, { + "id" : "access:Recommendation", + "name" : "Recommendation", + "type" : "query", + "stage" : "postgres", + "documentation" : "Recommend pages that are visited shortly after", + "inputs" : [ "default_catalog.default_database.Recommendation" ], + "annotations" : [ { + "name" : "parameters", + "description" : "url" + }, { + "name" : "base-table", + "description" : "Recommendation" + } ], + "plan" : "LogicalProject(url=[$0], rec=[$1], frequency=[$2])\n LogicalFilter(condition=[=($0, ?0)])\n LogicalTableScan(table=[[default_catalog, default_database, Recommendation]])\n", + "sql" : "SELECT *\nFROM `default_catalog`.`default_database`.`Recommendation`\nWHERE `url` = ?" +}, { + "id" : "access:Trending", + "name" : "Trending", + "type" : "query", + "stage" : "postgres", + "documentation" : "Most visited pages", + "inputs" : [ "default_catalog.default_database.Trending" ], + "annotations" : [ { + "name" : "base-table", + "description" : "Trending" + } ], + "plan" : "LogicalProject(url=[$0], total=[$1])\n LogicalTableScan(table=[[default_catalog, default_database, Trending]])\n", + "sql" : "SELECT *\nFROM `default_catalog`.`default_database`.`Trending`" +}, { + "id" : "access:VisitAfter", + "name" : "VisitAfter", + "type" : "query", + "stage" : "postgres", + "inputs" : [ "default_catalog.default_database.VisitAfter" ], + "annotations" : [ { + "name" : "base-table", + "description" : "VisitAfter" + } ], + "plan" : "LogicalProject(beforeURL=[$0], afterURL=[$1], timestamp=[$2])\n LogicalTableScan(table=[[default_catalog, default_database, VisitAfter]])\n", + "sql" : "SELECT *\nFROM `default_catalog`.`default_database`.`VisitAfter`" +}, { + "id" : "default_catalog.default_database.Click", + "name" : "Click", + "type" : "stream", + "stage" : "flink", + "inputs" : [ "default_catalog.default_database.Click__base" ], + "annotations" : [ { + "name" : "stream-root", + "description" : "Click" + } ], + "plan" : "LogicalWatermarkAssigner(rowtime=[timestamp], watermark=[-($1, 1000:INTERVAL SECOND)])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "sql" : "CREATE TEMPORARY TABLE `Click__schema` (\n `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL,\n `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL,\n `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL\n)\nWITH (\n 'connector' = 'datagen'\n);\nCREATE TABLE `Click` (\n PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED,\n WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND\n)\nWITH (\n 'format' = 'flexible-json',\n 'path' = '${DATA_PATH}/click.jsonl',\n 'source.monitor-interval' = '10 sec',\n 'connector' = 'filesystem'\n)\nLIKE `Click__schema`", + "timestamp" : "timestamp", + "schema" : [ { + "name" : "url", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + }, { + "name" : "timestamp", + "type" : "TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL" + }, { + "name" : "userid", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + } ], + "primary_key" : [ "url", "userid", "timestamp" ], + "row_count" : "~1e8" +}, { + "id" : "default_catalog.default_database.Click__base", + "name" : "Click", + "type" : "import", + "stage" : "flink", + "connector" : { + "format" : "flexible-json", + "path" : "${DATA_PATH}/click.jsonl", + "source.monitor-interval" : "10 sec", + "connector" : "filesystem" + } +}, { + "id" : "default_catalog.default_database.Recommendation", + "name" : "Recommendation", + "type" : "state", + "stage" : "flink", + "documentation" : "Recommend pages that are visited shortly after", + "inputs" : [ "default_catalog.default_database.VisitAfter" ], + "annotations" : [ { + "name" : "sort", + "description" : "[0 ASC-nulls-first, 2 DESC-nulls-last]" + } ], + "plan" : "LogicalAggregate(group=[{0, 1}], frequency=[COUNT()])\n LogicalProject(url=[$0], rec=[$1])\n LogicalTableScan(table=[[default_catalog, default_database, VisitAfter]])\n", + "sql" : "CREATE VIEW `Recommendation` AS SELECT beforeURL AS url, afterURL AS rec,\n count(1) AS frequency FROM VisitAfter\n GROUP BY beforeURL, afterURL ORDER BY url ASC, frequency DESC;\n", + "timestamp" : "-", + "schema" : [ { + "name" : "url", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + }, { + "name" : "rec", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL", + "description" : "the recommended page URL" + }, { + "name" : "frequency", + "type" : "BIGINT NOT NULL", + "description" : "the number of visitors that co-visited that page" + } ], + "primary_key" : [ "url", "rec" ], + "row_count" : "~2e7" +}, { + "id" : "default_catalog.default_database.Trending", + "name" : "Trending", + "type" : "state", + "stage" : "flink", + "documentation" : "Most visited pages", + "inputs" : [ "default_catalog.default_database.Click" ], + "annotations" : [ { + "name" : "sort", + "description" : "[1 DESC-nulls-last, 0 ASC-nulls-first]" + } ], + "plan" : "LogicalAggregate(group=[{0}], total=[COUNT()])\n LogicalProject(url=[$0])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "sql" : "CREATE VIEW `Trending` AS SELECT url, count(1) AS total\n FROM Click\n GROUP BY url ORDER BY total DESC, url ASC;\n", + "timestamp" : "-", + "schema" : [ { + "name" : "url", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL", + "description" : "URL of the top visited page" + }, { + "name" : "total", + "type" : "BIGINT NOT NULL", + "description" : "Total number of visitors" + } ], + "primary_key" : [ "url" ], + "row_count" : "~1e7" +}, { + "id" : "default_catalog.default_database.VisitAfter", + "name" : "VisitAfter", + "type" : "stream", + "stage" : "flink", + "inputs" : [ "default_catalog.default_database.Click" ], + "annotations" : [ ], + "plan" : "LogicalProject(beforeURL=[$0], afterURL=[$3], timestamp=[$4])\n LogicalJoin(condition=[AND(=($2, $5), <($1, $4), >=($1, -($4, *(10, 60000:INTERVAL MINUTE))))], joinType=[inner])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "sql" : "CREATE VIEW `VisitAfter` AS SELECT b.url AS beforeURL, a.url AS afterURL,\n a.`timestamp` AS `timestamp`\n FROM Click b JOIN Click a ON b.userid=a.userid AND\n b.`timestamp` < a.`timestamp` AND\n b.`timestamp` >= a.`timestamp` - INTERVAL 10 MINUTE;\n", + "timestamp" : "timestamp", + "schema" : [ { + "name" : "beforeURL", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + }, { + "name" : "afterURL", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + }, { + "name" : "timestamp", + "type" : "TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL" + } ], + "row_count" : "~3e7" +} ] +>>>pipeline_source.sqrl +CREATE TEMPORARY TABLE `Click__schema` ( + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `Click` ( + PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = '${DATA_PATH}/click.jsonl', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `Click__schema`; +/** Most visited pages + Columns: + - url: URL of the top visited page + - total: Total number of visitors +*/ +Trending := SELECT url, count(1) AS total + FROM Click + GROUP BY url ORDER BY total DESC, url ASC; +VisitAfter := SELECT b.url AS beforeURL, a.url AS afterURL, + a.`timestamp` AS `timestamp` + FROM Click b JOIN Click a ON b.userid=a.userid AND + b.`timestamp` < a.`timestamp` AND + b.`timestamp` >= a.`timestamp` - INTERVAL 10 MINUTE; +/** Recommend pages that are visited shortly after + Argument: + * url: the URL to get recommendations for + Columns: + * rec: the recommended page URL + * frequency: the number of visitors that co-visited that page +*/ +/*+query_by_all(url) */ +Recommendation := SELECT beforeURL AS url, afterURL AS rec, + count(1) AS frequency FROM VisitAfter + GROUP BY beforeURL, afterURL ORDER BY url ASC, frequency DESC; +/*+test */ +RankTest := SELECT url, count(1) AS total + FROM Click + GROUP BY url ORDER BY total DESC, url ASC; + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt new file mode 100644 index 0000000000..9d415f0b34 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt @@ -0,0 +1,535 @@ +>>>inferred_schema.graphqls +type Click { + url: String! + timestamp: DateTime! + userid: String! +} + +type ClickPage { + results: [Click!] + pagination: OffsetPageInfo +} + +"An RFC-3339 compliant Full Date Scalar" +scalar Date + +"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats" +scalar DateTime + +"A JSON scalar" +scalar JSON + +"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`." +scalar LocalTime + +"A 64-bit signed integer" +scalar Long + +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +type Query { + Click(limit: Int = 10, offset: Int = 0): ClickPage! + "Recommend pages that are visited shortly after" + Recommendation( + "the URL to get recommendations for" + url: String!, + limit: Int = 10, + offset: Int = 0 + ): RecommendationPage! + "Most visited pages" + Trending(limit: Int = 10, offset: Int = 0): TrendingPage! + VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage! +} + +"Recommend pages that are visited shortly after" +type Recommendation { + url: String! + "the recommended page URL" + rec: String! + "the number of visitors that co-visited that page" + frequency: Long! +} + +type RecommendationPage { + results: [Recommendation!] + pagination: OffsetPageInfo +} + +"Most visited pages" +type Trending { + "URL of the top visited page" + url: String! + "Total number of visitors" + total: Long! +} + +type TrendingPage { + results: [Trending!] + pagination: OffsetPageInfo +} + +type VisitAfter { + beforeURL: String! + afterURL: String! + timestamp: DateTime! +} + +type VisitAfterPage { + results: [VisitAfter!] + pagination: OffsetPageInfo +} + +enum _McpMethodType { + NONE + TOOL + RESOURCE +} + +enum _RestMethodType { + NONE + GET + POST +} + +directive @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION + +>>>pipeline_explain.txt +=== Click +ID: default_catalog.default_database.Click +Type: stream +Stage: flink +Primary key: url, userid, timestamp +Timestamp: timestamp +Row count: ~1e8 +--- +Schema: + - url: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL + - userid: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL +Inputs: + - default_catalog.default_database.Click__base +Annotations: + - stream-root: Click + +=== Recommendation +ID: default_catalog.default_database.Recommendation +Type: state +Stage: flink +Primary key: url, rec +Timestamp: - +Row count: ~2e7 +--- +Schema: + - url: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - rec: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - frequency: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.VisitAfter +Annotations: + - sort: [0 ASC-nulls-first, 2 DESC-nulls-last] + +=== Trending +ID: default_catalog.default_database.Trending +Type: state +Stage: flink +Primary key: url +Timestamp: - +Row count: ~1e7 +--- +Schema: + - url: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - total: BIGINT NOT NULL +Inputs: + - default_catalog.default_database.Click +Annotations: + - sort: [1 DESC-nulls-last, 0 ASC-nulls-first] + +=== VisitAfter +ID: default_catalog.default_database.VisitAfter +Type: stream +Stage: flink +Primary key: - +Timestamp: timestamp +Row count: ~3e7 +--- +Schema: + - beforeURL: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - afterURL: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL +Inputs: + - default_catalog.default_database.Click + +>>>flink-sql-no-functions.sql +CREATE TEMPORARY TABLE `Click__schema` ( + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL +) +WITH ( + 'connector' = 'datagen' +); +CREATE TABLE `Click` ( + PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED, + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND +) +WITH ( + 'format' = 'flexible-json', + 'path' = '${DATA_PATH}/click.jsonl', + 'source.monitor-interval' = '10 sec', + 'connector' = 'filesystem' +) +LIKE `Click__schema`; +CREATE VIEW `Trending` +AS +SELECT `url`, COUNT(1) AS `total` +FROM `Click` +GROUP BY `url`; +CREATE VIEW `VisitAfter` +AS +SELECT `b`.`url` AS `beforeURL`, `a`.`url` AS `afterURL`, `a`.`timestamp` AS `timestamp` +FROM `Click` AS `b` + INNER JOIN `Click` AS `a` ON `b`.`userid` = `a`.`userid` AND `b`.`timestamp` < `a`.`timestamp` AND `b`.`timestamp` >= `a`.`timestamp` - (INTERVAL 10 MINUTE); +CREATE VIEW `Recommendation` +AS +SELECT `beforeURL` AS `url`, `afterURL` AS `rec`, COUNT(1) AS `frequency` +FROM `VisitAfter` +GROUP BY `beforeURL`, `afterURL`; +CREATE TABLE `Click_1` ( + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'Click', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Recommendation_2` ( + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `rec` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `frequency` BIGINT NOT NULL, + PRIMARY KEY (`url`, `rec`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'table-name' = 'Recommendation', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `Trending_3` ( + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `total` BIGINT NOT NULL, + PRIMARY KEY (`url`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'table-name' = 'Trending', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +CREATE TABLE `VisitAfter_4` ( + `beforeURL` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `afterURL` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, + `__pk_hash` CHAR(32) CHARACTER SET `UTF-16LE`, + PRIMARY KEY (`__pk_hash`) NOT ENFORCED +) +WITH ( + 'connector' = 'jdbc-sqrl', + 'driver' = 'org.postgresql.Driver', + 'password' = '${POSTGRES_PASSWORD}', + 'sink.on-conflict.action' = 'IGNORE', + 'table-name' = 'VisitAfter', + 'url' = 'jdbc:postgresql://${POSTGRES_AUTHORITY}', + 'username' = '${POSTGRES_USERNAME}' +); +EXECUTE STATEMENT SET BEGIN +INSERT INTO `default_catalog`.`default_database`.`Click_1` +SELECT * +FROM `default_catalog`.`default_database`.`Click` +; +INSERT INTO `default_catalog`.`default_database`.`Recommendation_2` +SELECT * +FROM `default_catalog`.`default_database`.`Recommendation` +; +INSERT INTO `default_catalog`.`default_database`.`Trending_3` +SELECT * +FROM `default_catalog`.`default_database`.`Trending` +; +INSERT INTO `default_catalog`.`default_database`.`VisitAfter_4` +SELECT `beforeURL`, `afterURL`, `timestamp`, `hash_columns`(`beforeURL`, `afterURL`, `timestamp`) AS `__pk_hash` +FROM `default_catalog`.`default_database`.`VisitAfter` +; +END; + +>>>postgres-schema.sql +CREATE TABLE IF NOT EXISTS "Click" ("url" TEXT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "userid" TEXT NOT NULL, PRIMARY KEY ("url","userid","timestamp")); +CREATE TABLE IF NOT EXISTS "Recommendation" ("url" TEXT NOT NULL, "rec" TEXT NOT NULL, "frequency" BIGINT NOT NULL, PRIMARY KEY ("url","rec")); +CREATE TABLE IF NOT EXISTS "Trending" ("url" TEXT NOT NULL, "total" BIGINT NOT NULL, PRIMARY KEY ("url")); +CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" TEXT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); + +>>>vertx.json +{ + "models" : { + "v1" : { + "queries" : [ + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "Click", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM \"Click\"", + "parameters" : [ ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Click\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x" + } + } + }, + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "Recommendation", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + }, + { + "type" : "variable", + "path" : "url" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM (SELECT \"url\", \"rec\", \"frequency\"\n FROM \"Recommendation\"\n ORDER BY \"url\" NULLS FIRST, \"frequency\" DESC NULLS LAST) AS \"t\"\nWHERE \"url\" = $1", + "parameters" : [ + { + "type" : "arg", + "path" : "url", + "sqlType" : "VARCHAR" + } + ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"rec\", \"frequency\"\n FROM \"Recommendation\"\n ORDER BY \"url\" NULLS FIRST, \"frequency\" DESC NULLS LAST) AS \"t\"\nWHERE \"url\" = $1) x" + } + } + }, + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "Trending", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT *\nFROM (SELECT \"url\", \"total\"\n FROM \"Trending\"\n ORDER BY \"total\" DESC NULLS LAST, \"url\" NULLS FIRST) AS \"t\"", + "parameters" : [ ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"total\"\n FROM \"Trending\"\n ORDER BY \"total\" DESC NULLS LAST, \"url\" NULLS FIRST) AS \"t\") x" + } + } + }, + { + "type" : "args", + "parentType" : "Query", + "fieldName" : "VisitAfter", + "exec" : { + "arguments" : [ + { + "type" : "variable", + "path" : "limit" + }, + { + "type" : "variable", + "path" : "offset" + } + ], + "query" : { + "type" : "SqlQuery", + "sql" : "SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\"", + "parameters" : [ ], + "pagination" : "LIMIT_AND_OFFSET", + "cacheDurationMs" : 0, + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" + } + } + } + ], + "mutations" : [ ], + "subscriptions" : [ ], + "operations" : [ + { + "function" : { + "name" : "GetClick", + "parameters" : { + "type" : "object", + "properties" : { + "offset" : { + "type" : "integer" + }, + "limit" : { + "type" : "integer" + } + }, + "required" : [ ] + } + }, + "format" : "JSON", + "apiQuery" : { + "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\nurl\ntimestamp\nuserid\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "queryName" : "Click", + "operationType" : "QUERY" + }, + "mcpMethod" : "TOOL", + "restMethod" : "GET", + "uriTemplate" : "queries/Click{?offset,limit}" + }, + { + "function" : { + "name" : "GetRecommendation", + "description" : "Recommend pages that are visited shortly after", + "parameters" : { + "type" : "object", + "properties" : { + "offset" : { + "type" : "integer" + }, + "limit" : { + "type" : "integer" + }, + "url" : { + "type" : "string", + "description" : "the URL to get recommendations for" + } + }, + "required" : [ + "url" + ] + } + }, + "format" : "JSON", + "apiQuery" : { + "query" : "query Recommendation($url: String!, $limit: Int = 10, $offset: Int = 0) {\nRecommendation(url: $url, limit: $limit, offset: $offset) {\nresults {\nurl\nrec\nfrequency\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "queryName" : "Recommendation", + "operationType" : "QUERY" + }, + "mcpMethod" : "TOOL", + "restMethod" : "GET", + "uriTemplate" : "queries/Recommendation{?offset,limit,url}" + }, + { + "function" : { + "name" : "GetTrending", + "description" : "Most visited pages", + "parameters" : { + "type" : "object", + "properties" : { + "offset" : { + "type" : "integer" + }, + "limit" : { + "type" : "integer" + } + }, + "required" : [ ] + } + }, + "format" : "JSON", + "apiQuery" : { + "query" : "query Trending($limit: Int = 10, $offset: Int = 0) {\nTrending(limit: $limit, offset: $offset) {\nresults {\nurl\ntotal\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "queryName" : "Trending", + "operationType" : "QUERY" + }, + "mcpMethod" : "TOOL", + "restMethod" : "GET", + "uriTemplate" : "queries/Trending{?offset,limit}" + }, + { + "function" : { + "name" : "GetVisitAfter", + "parameters" : { + "type" : "object", + "properties" : { + "offset" : { + "type" : "integer" + }, + "limit" : { + "type" : "integer" + } + }, + "required" : [ ] + } + }, + "format" : "JSON", + "apiQuery" : { + "query" : "query VisitAfter($limit: Int = 10, $offset: Int = 0) {\nVisitAfter(limit: $limit, offset: $offset) {\nresults {\nbeforeURL\nafterURL\ntimestamp\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "queryName" : "VisitAfter", + "operationType" : "QUERY" + }, + "mcpMethod" : "TOOL", + "restMethod" : "GET", + "uriTemplate" : "queries/VisitAfter{?offset,limit}" + } + ], + "schema" : { + "type" : "string", + "schema" : "type Click {\n url: String!\n timestamp: DateTime!\n userid: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" + } + } + } +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/usecases/clickstream/package-paginated.json b/sqrl-testing/sqrl-testing-integration/src/test/resources/usecases/clickstream/package-paginated.json new file mode 100644 index 0000000000..d84685ffad --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/usecases/clickstream/package-paginated.json @@ -0,0 +1,17 @@ +{ + "version": "1", + "enabled-engines": ["vertx", "postgres", "flink"], + "compiler": { + "api": { + "paginated-results": true + } + }, + "script": { + "main": "clickstream-teaser.sqrl" + }, + "test-runner": { + "snapshot-folder": "snapshots-clickstream-teaser", + "test-folder": "tests-clickstream-teaser", + "delay-sec": -1 + } +} From d072d3cc80178d91d08988e4bba19a517ba9e5ab Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Thu, 9 Jul 2026 13:59:34 -0300 Subject: [PATCH 05/16] refactor: Require users to declare OffsetPageInfo instead of auto-injecting it Signed-off-by: Marvin Froeder --- .../datasqrl/config/GraphqlSourceLoader.java | 15 +--- .../server/GraphqlSchemaValidator.java | 2 +- .../datasqrl/server/OffsetPageInfoUtil.java | 74 ++++--------------- ...veTest-fail-paged-no-limit-offset.graphqls | 13 ++++ ...hensiveTest-fail-paged-undeclared.graphqls | 21 ++++++ ...siveTest-fail-paged-unknown-field.graphqls | 13 ++++ .../comprehensiveTest-paged-results.graphqls | 13 ++++ ...hensiveTest-fail-paged-no-limit-offset.txt | 2 +- ...omprehensiveTest-fail-paged-undeclared.txt | 41 ++++++++++ ...ehensiveTest-limit-offset-combinations.txt | 62 ++++++++++++++++ .../comprehensiveTest-paged-results.txt | 64 +++++++++++++++- .../comprehensiveTest-paged-userdefined.txt | 62 ++++++++++++++++ .../comprehensiveTest-parameters-order.txt | 62 ++++++++++++++++ .../comprehensiveTest.txt | 62 ++++++++++++++++ 14 files changed, 432 insertions(+), 74 deletions(-) create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-undeclared.graphqls create mode 100644 sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java index 72c9f3e1d8..85a67c8dd2 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java @@ -23,7 +23,6 @@ import com.datasqrl.server.ApiSource; import com.datasqrl.server.ApiSources; import com.datasqrl.server.GraphqlSchemaHandler; -import com.datasqrl.server.OffsetPageInfoUtil; import com.datasqrl.server.ScriptFiles; import java.nio.file.Files; import java.nio.file.Path; @@ -76,17 +75,9 @@ public LoadResult load(ServerPhysicalPlan serverPlan) { } if (!shouldUseInferredSchema(apiVersions)) { - var injected = - apiVersions.stream() - .map( - apiVersion -> - new ApiSources( - apiVersion.version(), - OffsetPageInfoUtil.injectPaginationType(apiVersion.schema()), - apiVersion.operations())) - .toList(); - injected.forEach(apiVersion -> graphqlSchemaHandler.validateSchema(apiVersion, serverPlan)); - return new LoadResult(injected, Optional.empty()); + apiVersions.forEach( + apiVersion -> graphqlSchemaHandler.validateSchema(apiVersion, serverPlan)); + return new LoadResult(apiVersions, Optional.empty()); } List operations; diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java index a3c4c0d6e4..1bb910202f 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaValidator.java @@ -378,7 +378,7 @@ protected void visitQuery( boolean paged) { checkValidArrayNonNullType(atField.getType()); if (paged) { - OffsetPageInfoUtil.validatePaginationType(registry); + OffsetPageInfoUtil.validatePaginationType(registry, atField.getSourceLocation()); var argNames = atField.getInputValueDefinitions().stream() .map(InputValueDefinition::getName) diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index 8c19674d9d..728f2e3a6c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -23,13 +23,13 @@ import graphql.language.ListType; import graphql.language.NonNullType; import graphql.language.ObjectTypeDefinition; +import graphql.language.SourceLocation; import graphql.language.Type; import graphql.language.TypeName; import graphql.schema.GraphQLFieldDefinition; import graphql.schema.GraphQLNonNull; import graphql.schema.GraphQLObjectType; import graphql.schema.GraphQLOutputType; -import graphql.schema.idl.SchemaParser; import graphql.schema.idl.TypeDefinitionRegistry; import java.util.LinkedHashMap; import java.util.Map; @@ -38,8 +38,8 @@ /** * Opt-in pagination support: a query whose result type is a page wrapper ({@code {results: * [Element!] pagination: OffsetPageInfo}}) returns its rows plus pagination metadata computed from - * a companion COUNT(*) query. This util detects the wrapper shape and injects/validates the - * standard {@code OffsetPageInfo} type. + * a companion COUNT(*) query. This util detects the wrapper shape and validates the user-declared + * {@code OffsetPageInfo} type. */ public final class OffsetPageInfoUtil { @@ -94,48 +94,20 @@ private static String buildCanonicalSdl() { } /** - * If the schema references {@code OffsetPageInfo} but does not define it, append the canonical - * definition (plus any missing scalar declarations). A user-provided definition is left untouched - * here and validated later by {@link #validatePaginationType} within the schema validator's error - * scope. Returns the (possibly rewritten) source. + * Validates that a paginated query's schema declares the {@code OffsetPageInfo} type and that it + * matches the canonical definition. Users must declare the type themselves; we only validate it. + * Must be called from within the schema validator so errors are reported like other schema + * errors. */ - public static ApiSource injectPaginationType(ApiSource schema) { - TypeDefinitionRegistry registry; - try { - registry = new SchemaParser().parse(schema.getDefinition()); - } catch (Exception e) { - // Let the downstream validator report parse errors with proper source location; an - // unparseable schema cannot reference the pagination type anyway. - return schema; - } - - if (!referencesPaginationType(registry) || registry.getType(PAGINATION_TYPE_NAME).isPresent()) { - return schema; - } - - var injected = new StringBuilder(schema.getDefinition()); - injected.append("\n"); - if (registry.scalars().get("Long") == null) { - injected.append("scalar Long\n"); - } - if (registry.scalars().get("DateTime") == null) { - injected.append("scalar DateTime\n"); - } - injected.append(CANONICAL_SDL); - - return new ApiSource(schema.getPath().orElse(null), injected.toString()); - } - - /** - * Validates that a user-provided {@code OffsetPageInfo} type matches the canonical definition. - * Must be called from within the schema validator so mismatches are reported like other schema - * errors. No-op when the type is absent (it will have been injected) or unreferenced. - */ - public static void validatePaginationType(TypeDefinitionRegistry registry) { + public static void validatePaginationType( + TypeDefinitionRegistry registry, SourceLocation location) { var existing = registry.getType(PAGINATION_TYPE_NAME); - if (existing.isEmpty()) { - return; - } + checkState( + existing.isPresent(), + location, + "Paginated results require the %s type to be declared in the schema:\n%s", + PAGINATION_TYPE_NAME, + CANONICAL_SDL); checkState( existing.get() instanceof ObjectTypeDefinition, existing.get().getSourceLocation(), @@ -179,22 +151,6 @@ public static Optional getPagedElementType( return hasPagination ? Optional.ofNullable(elementType) : Optional.empty(); } - private static boolean referencesPaginationType(TypeDefinitionRegistry registry) { - return registry.types().values().stream() - .filter(t -> t instanceof ObjectTypeDefinition) - .flatMap(t -> ((ObjectTypeDefinition) t).getFieldDefinitions().stream()) - .anyMatch(field -> referencesPaginationType(field.getType())); - } - - private static boolean referencesPaginationType(Type type) { - var unwrapped = unwrapNonNull(type); - if (unwrapped instanceof ListType listType) { - return referencesPaginationType(listType.getType()); - } - return unwrapped instanceof TypeName typeName - && PAGINATION_TYPE_NAME.equals(typeName.getName()); - } - private static void validateMatchesCanonical(ObjectTypeDefinition userType) { var actual = new LinkedHashMap(); for (FieldDefinition field : userType.getFieldDefinitions()) { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls index e9b95869e3..644c750d93 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls @@ -16,6 +16,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-undeclared.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-undeclared.graphqls new file mode 100644 index 0000000000..fce97c9ce1 --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-undeclared.graphqls @@ -0,0 +1,21 @@ +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls index 1c48801473..5645b9cc2d 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls @@ -17,6 +17,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls index fce97c9ce1..8b711ba362 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls @@ -16,6 +16,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt index dbdfcf10cc..833ec77664 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt @@ -20,7 +20,7 @@ CustomerTimeWindow := SELECT ^ [FATAL] Paginated query [CustomerByTime2] must declare both 'limit' and 'offset' arguments -in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [20:5]: +in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [33:5]: type Query { CustomerByTime2: CustomerPage! diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt new file mode 100644 index 0000000000..74602170fe --- /dev/null +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt @@ -0,0 +1,41 @@ +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database._Customer[timestamp] +in script:comprehensiveTest.sqrl [10:1]: +CustomerFilteredDistinct := DISTINCT Customer ON customerid ORDER BY lastUpdated DESC; + +AnotherCustomer := SELECT customerid, email, lastUpdated FROM _Customer WHERE customerid > 100; +^ + +[NOTICE] You can rewrite the join as a temporal join for greater efficiency by adding: FOR SYSTEM_TIME AS OF `time` +in script:comprehensiveTest.sqrl [18:1]: +InvalidDistinct := SELECT customerid, `timestamp`, name AS namee FROM (SELECT *, (ROW_NUMBER() OVER (PARTITION BY customerid ORDER BY `timestamp` DESC)) AS _rownum FROM Customer) WHERE (_rownum = 1); + +MissedTemporalJoin := SELECT * FROM ExternalOrders o JOIN ExplicitDistinct c ON o.customerid = c.customerid; +^ + +[NOTICE] This table does not propagate the source row time columns: default_catalog.default_database.SelectCustomers[timestamp] +in script:comprehensiveTest.sqrl [51:1]: +); + +CustomerTimeWindow := SELECT +^ + +[FATAL] Paginated results require the OffsetPageInfo type to be declared in the schema: +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + +in script:comprehensiveTest-fail-paged-undeclared.graphqls [20:5]: + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +----^ + diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt index 084566fe4d..ca13edbb83 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt @@ -288,6 +288,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } @@ -319,6 +332,29 @@ type Subscription { CustomerSubscription: CustomerPage } +>>>comprehensiveTest-fail-paged-undeclared.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-paged-unknown-field.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -339,6 +375,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -538,6 +587,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index 436f8e9224..69faa45aa6 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -288,6 +288,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } @@ -319,6 +332,29 @@ type Subscription { CustomerSubscription: CustomerPage } +>>>comprehensiveTest-fail-paged-undeclared.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-paged-unknown-field.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -339,6 +375,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -538,6 +587,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -1816,7 +1878,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 39a8ec0209..0aaf2c981d 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -288,6 +288,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } @@ -319,6 +332,29 @@ type Subscription { CustomerSubscription: CustomerPage } +>>>comprehensiveTest-fail-paged-undeclared.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-paged-unknown-field.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -339,6 +375,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -538,6 +587,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt index d2fd49c7b0..2c386dbb74 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt @@ -288,6 +288,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } @@ -319,6 +332,29 @@ type Subscription { CustomerSubscription: CustomerPage } +>>>comprehensiveTest-fail-paged-undeclared.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-paged-unknown-field.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -339,6 +375,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -538,6 +587,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt index 8d8fec4e4c..2565c3622c 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt @@ -288,6 +288,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2: CustomerPage! } @@ -319,6 +332,29 @@ type Subscription { CustomerSubscription: CustomerPage } +>>>comprehensiveTest-fail-paged-undeclared.graphqls +"An RFC-3339 compliant DateTime Scalar" +scalar DateTime +"A 64-bit signed integer" +scalar Long + +type Customer { + customerid: Long! + email: String! + name: String! + lastUpdated: Long! + timestamp: DateTime! +} + +type CustomerPage { + results: [Customer!] + pagination: OffsetPageInfo +} + +type Query { + CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! +} + >>>comprehensiveTest-fail-paged-unknown-field.graphqls "An RFC-3339 compliant DateTime Scalar" scalar DateTime @@ -339,6 +375,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } @@ -538,6 +587,19 @@ type CustomerPage { pagination: OffsetPageInfo } +type OffsetPageInfo { + totalRecords: Long! + pageSize: Int! + currentPage: Int! + totalPages: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} + type Query { CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage! } From 990820ad13322e3487223779a60fd1eaac569347 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Thu, 9 Jul 2026 14:38:07 -0300 Subject: [PATCH 06/16] fix: Avoid NPE on null pagination bind params and report real pageSize when limit is unbounded Signed-off-by: Marvin Froeder --- .../jdbc/VertxQueryExecutionContext.java | 12 +++++-- .../com/datasqrl/server/PagedQueryIT.java | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index 314a052c3b..5c9b92bf80 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -104,9 +104,10 @@ private CompletableFuture runQueryInternal( && selection.containsAnyOf(pag + "/hasNextPage", pag + "/nextOffset"); } - // The aggregate query is bound with the base parameters only, without the runtime - // limit/offset. - var countParams = aggregateSql != null ? Tuple.from(List.copyOf(paramObj)) : null; + // The aggregate query binds the base parameters only, without the runtime limit/offset. + // Tuple.from snapshots the list here (before limit/offset are appended below) and, unlike + // List.copyOf, tolerates null bind values (SQL NULL). + var countParams = aggregateSql != null ? Tuple.from(paramObj) : null; int limitValue = Integer.MAX_VALUE; int offsetValue = 0; @@ -238,6 +239,11 @@ private Object pagedResultMapper( var pagination = buildPaginationMetadata( totalRecords, hasNextPage, limit, offset, firstEventTime, lastEventTime); + // An absent limit means "return everything" (limit == Integer.MAX_VALUE); report the actual + // number of rows on this page as pageSize rather than leaking the sentinel. + if (limit == Integer.MAX_VALUE) { + pagination.put("pageSize", results.size()); + } return new JsonObject() .put(fieldNames.resultsField(), results) .put(fieldNames.paginationField(), pagination); diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java index 42f58b5d8a..402ce638f0 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -222,6 +222,21 @@ void givenEventTimesSelected_whenQuery_thenMinMaxVariantRuns() { assertThat(String.valueOf(pagination.get("lastEventTime"))).startsWith("2024-01-05"); } + @Test + void givenNoLimitArgument_whenQuery_thenPageSizeReportsRowCountNotSentinel() { + var customers = + execute( + "{ customers: customersUnbounded { results { customerid }" + + " pagination { pageSize totalRecords hasNextPage } } }"); + + assertThat(results(customers)).hasSize(5); + // an absent limit fetches every row; pageSize reflects the rows returned, not Integer.MAX_VALUE + assertThat(pagination(customers)) + .containsEntry("pageSize", 5) + .containsEntry("totalRecords", 5L) + .containsEntry("hasNextPage", false); + } + @SneakyThrows private Map execute(String query) { var result = graphQL.execute(ExecutionInput.newExecutionInput().query(query).build()); @@ -254,6 +269,7 @@ private RootGraphQLModel getPagedModel() { scalar Long type Query { customers(limit: Int = 10, offset: Int = 0): CustomerPage! + customersUnbounded(limit: Int, offset: Int = 0): CustomerPage! } type Customer { customerid: Int @@ -293,6 +309,23 @@ private RootGraphQLModel getPagedModel() { COUNT_WITH_EVENT_TIMES_SQL)) .build()) .build()) + .query( + ArgumentLookupQueryCoords.builder() + .parentType("Query") + .fieldName("customersUnbounded") + .exec( + QueryWithArguments.builder() + .query( + new SqlQuery( + BASE_SQL, + List.of(), + PaginationType.LIMIT_AND_OFFSET, + 0, + DatabaseType.POSTGRES, + COUNT_SQL, + COUNT_WITH_EVENT_TIMES_SQL)) + .build()) + .build()) .build(); } From 35269acb61dca6344fecba1a4af3f46dfae21f98 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Thu, 9 Jul 2026 14:45:25 -0300 Subject: [PATCH 07/16] test: Cover paginated event-times selection when the result has no rowtime column Signed-off-by: Marvin Froeder --- .../com/datasqrl/server/PagedQueryIT.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java index 402ce638f0..8f0e1888eb 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -237,6 +237,23 @@ void givenNoLimitArgument_whenQuery_thenPageSizeReportsRowCountNotSentinel() { .containsEntry("hasNextPage", false); } + @Test + void givenEventTimesSelectedButNoRowtime_whenQuery_thenPlainCountRunsAndEventTimesAreNull() { + var customers = + execute( + "{ customers: customersNoRowtime(limit: 2) { results { customerid }" + + " pagination { totalRecords firstEventTime lastEventTime } } }"); + + assertThat(recordingClient.executed).hasSize(2); + // no rowtime => the MIN/MAX variant is absent; the plain count runs and event times stay null + assertThat(sqlOf(recordingClient.executed)).contains(COUNT_SQL); + assertThat(sqlOf(recordingClient.executed)).doesNotContain(COUNT_WITH_EVENT_TIMES_SQL); + var pagination = pagination(customers); + assertThat(pagination).containsEntry("totalRecords", 5L); + assertThat(pagination.get("firstEventTime")).isNull(); + assertThat(pagination.get("lastEventTime")).isNull(); + } + @SneakyThrows private Map execute(String query) { var result = graphQL.execute(ExecutionInput.newExecutionInput().query(query).build()); @@ -270,6 +287,7 @@ private RootGraphQLModel getPagedModel() { type Query { customers(limit: Int = 10, offset: Int = 0): CustomerPage! customersUnbounded(limit: Int, offset: Int = 0): CustomerPage! + customersNoRowtime(limit: Int = 10, offset: Int = 0): CustomerPage! } type Customer { customerid: Int @@ -326,6 +344,23 @@ private RootGraphQLModel getPagedModel() { COUNT_WITH_EVENT_TIMES_SQL)) .build()) .build()) + .query( + ArgumentLookupQueryCoords.builder() + .parentType("Query") + .fieldName("customersNoRowtime") + .exec( + QueryWithArguments.builder() + .query( + new SqlQuery( + BASE_SQL, + List.of(), + PaginationType.LIMIT_AND_OFFSET, + 0, + DatabaseType.POSTGRES, + COUNT_SQL, + null)) + .build()) + .build()) .build(); } From 6d7e26c414f3e71c76ba9ac6438a6ae1b7848585 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Fri, 10 Jul 2026 11:01:15 -0300 Subject: [PATCH 08/16] refactor: Replace count-based pagination with OFFSET_PAGE_INFO and index paged rowtime columns Signed-off-by: Marvin Froeder --- .../datasqrl/compile/CompilationProcess.java | 12 + .../engine/server/ServerPhysicalPlan.java | 9 + .../global/PagedRowtimeIndexRewriter.java | 93 +++++++ .../datasqrl/server/GenerateServerModel.java | 1 + .../server/GraphqlModelGenerator.java | 35 +-- .../datasqrl/server/OffsetPageInfoUtil.java | 15 +- .../com/datasqrl/server/PaginationType.java | 6 +- .../server/graphql/GraphQLEngineBuilder.java | 3 + .../server/graphql/RootGraphQLModel.java | 25 +- .../jdbc/VertxQueryExecutionContext.java | 252 +++++++++--------- .../com/datasqrl/server/PagedQueryIT.java | 71 ++--- .../java/com/datasqrl/server/WriteIT.java | 1 - .../server/jdbc/PaginationMetadataTest.java | 64 +---- ...veTest-fail-paged-no-limit-offset.graphqls | 2 - ...siveTest-fail-paged-unknown-field.graphqls | 2 - .../comprehensiveTest-paged-results.graphqls | 2 - ...mprehensiveTest-paged-userdefined.graphqls | 2 - ...hensiveTest-fail-paged-no-limit-offset.txt | 2 +- ...omprehensiveTest-fail-paged-undeclared.txt | 2 - ...eTest-fail-paged-wrong-pagination-type.txt | 2 - ...ehensiveTest-limit-offset-combinations.txt | 8 - .../comprehensiveTest-paged-results.txt | 38 ++- .../comprehensiveTest-paged-userdefined.txt | 42 ++- .../comprehensiveTest-parameters-order.txt | 8 - .../comprehensiveTest.txt | 8 - .../clickstream-package-paginated.txt | 39 ++- 26 files changed, 356 insertions(+), 388 deletions(-) create mode 100644 sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java diff --git a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java index 5d6134015f..b337962bdb 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java +++ b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java @@ -26,6 +26,7 @@ import com.datasqrl.error.ErrorCode; import com.datasqrl.error.ErrorCollector; import com.datasqrl.plan.MainScript; +import com.datasqrl.plan.global.PagedRowtimeIndexRewriter; import com.datasqrl.plan.global.PhysicalPlanRewriter; import com.datasqrl.plan.validate.ExecutionGoal; import com.datasqrl.planner.SqlScriptPlanner; @@ -105,6 +106,17 @@ public Pair executeCompilation(Optional testsPath) serverPlan.getModels().put(api.version(), model); }); + // Paginated queries run a MIN/MAX(rowtime) aggregate; index their base tables' rowtime + // column. + // Which queries are paginated is only known now (after the GraphQL walk), so this runs as a + // second rewrite pass over the already-planned database DDL. + if (!serverPlan.getPagedRowtimeTables().isEmpty()) { + physicalPlan = + physicalPlan.applyRewriting( + List.of(new PagedRowtimeIndexRewriter(serverPlan.getPagedRowtimeTables())), + environment); + } + // create test artifact if (executionGoal == ExecutionGoal.TEST) { var gqlGenerator = new GqlGenerator(serverPlan.getFunctions()); diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java b/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java index 7331cc761f..1c8a8e83ce 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java @@ -16,13 +16,16 @@ package com.datasqrl.engine.server; import com.datasqrl.engine.EnginePhysicalPlan; +import com.datasqrl.planner.analyzer.TableAnalysis; import com.datasqrl.planner.dag.plan.MutationTable; import com.datasqrl.planner.tables.SqrlTableFunction; import com.datasqrl.server.graphql.RootGraphQLModel; import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -44,4 +47,10 @@ public class ServerPhysicalPlan implements EnginePhysicalPlan { * plan later. */ final Map models = new LinkedHashMap<>(); + + /** + * Base tables of paginated queries, collected during model generation so a rowtime index can be + * added to their physical tables afterwards. + */ + @JsonIgnore final Set pagedRowtimeTables = new LinkedHashSet<>(); } diff --git a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java new file mode 100644 index 0000000000..569ff1c95d --- /dev/null +++ b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.plan.global; + +import com.datasqrl.engine.EnginePhysicalPlan; +import com.datasqrl.engine.database.relational.AbstractJDBCDatabaseEngine; +import com.datasqrl.engine.database.relational.JdbcPhysicalPlan; +import com.datasqrl.engine.database.relational.JdbcStatement; +import com.datasqrl.planner.Sqrl2FlinkSQLTranslator; +import com.datasqrl.planner.analyzer.TableAnalysis; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; + +/** + * Adds a btree index on the rowtime column of every physical table that backs a paginated ({@code + * OffsetPageInfo}) query. Those queries run a companion {@code MIN/MAX(rowtime)} aggregate for + * their {@code first/lastEventTime}; the btree lets the database answer it from the index endpoints + * instead of scanning the table. + * + *

Unlike {@link JdbcIndexOptimization} this rewriter is constructed with the set of paginated + * base tables (only known after the GraphQL schema walk) and is applied in a second pass. + */ +@RequiredArgsConstructor +public class PagedRowtimeIndexRewriter implements PhysicalPlanRewriter { + + private final Set pagedBaseTables; + + @Override + public boolean appliesTo(EnginePhysicalPlan plan) { + return !pagedBaseTables.isEmpty() + && plan instanceof JdbcPhysicalPlan jpp + && jpp.stage().engine() instanceof AbstractJDBCDatabaseEngine; + } + + @Override + public JdbcPhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator sqrlEnv) { + var jdbcPlan = (JdbcPhysicalPlan) plan; + var engine = (AbstractJDBCDatabaseEngine) jdbcPlan.stage().engine(); + if (!engine.getIndexSelectorConfig().supportedIndexTypes().contains(IndexType.BTREE)) { + return jdbcPlan; + } + var stmtFactory = engine.getStatementFactory(); + // Existing indexes (e.g. from JdbcIndexOptimization) may already cover the rowtime column. + var existingIndexNames = + jdbcPlan.getStatementsForType(JdbcStatement.Type.INDEX).stream() + .map(JdbcStatement::getName) + .collect(Collectors.toSet()); + + var builder = jdbcPlan.toBuilder(); + for (var createTbl : jdbcPlan.tableIdMap().values()) { + var engineTable = createTbl.getEngineTable(); + var tableAnalysis = engineTable.tableAnalysis(); + if (!isPaged(tableAnalysis)) { + continue; + } + var rowTime = tableAnalysis.getRowTime(); + if (rowTime.isEmpty()) { + continue; + } + var index = + new IndexDefinition( + engineTable.tableName(), + List.of(rowTime.get()), + tableAnalysis.getRowType().getFieldNames(), + -1, + IndexType.BTREE); + if (existingIndexNames.add(index.getName())) { + builder.statement(stmtFactory.addIndex(index)); + } + } + return builder.build(); + } + + private boolean isPaged(TableAnalysis tableAnalysis) { + return pagedBaseTables.contains(tableAnalysis) + || pagedBaseTables.contains(tableAnalysis.getBaseTable()); + } +} diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java b/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java index 0ef405df0c..764228e69f 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java @@ -48,6 +48,7 @@ public RootGraphQLModel generateGraphQLModel(ApiSources api, ServerPhysicalPlan new GraphqlModelGenerator( serverPlan.getFunctions(), serverPlan.getMutations(), errorCollector); graphqlModelGenerator.walkAPISource(api.schema()); + serverPlan.getPagedRowtimeTables().addAll(graphqlModelGenerator.getPagedRowtimeTables()); var schema = StringSchema.builder().schema(api.schema().getDefinition()).build(); var graphSchema = converter.getSchema(schema.getSchema()); var apiConfig = configuration.getCompilerConfig().getApiConfig(); diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index aa10214f87..dd0990f92e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -22,6 +22,7 @@ import com.datasqrl.engine.log.kafka.KafkaLogEngine; import com.datasqrl.engine.log.kafka.KafkaQuery; import com.datasqrl.error.ErrorCollector; +import com.datasqrl.planner.analyzer.TableAnalysis; import com.datasqrl.planner.dag.plan.MutationTable; import com.datasqrl.planner.parser.AccessModifier; import com.datasqrl.planner.tables.SqrlFunctionParameter; @@ -50,6 +51,7 @@ import graphql.language.ObjectTypeDefinition; import graphql.schema.idl.TypeDefinitionRegistry; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -68,6 +70,10 @@ public class GraphqlModelGenerator extends GraphqlSchemaWalker { List queryCoords = new ArrayList<>(); List mutations = new ArrayList<>(); List subscriptions = new ArrayList<>(); + + /** Base tables of paginated queries, so a rowtime index can be generated for them. */ + Set pagedRowtimeTables = new LinkedHashSet<>(); + private final ErrorCollector errorCollector; public GraphqlModelGenerator( @@ -169,18 +175,23 @@ protected void visitQuery( .map(InputValueDefinition::getName) .anyMatch( name -> name.equals(SchemaConstants.LIMIT) || name.equals(SchemaConstants.OFFSET)); - var countSql = paged ? buildCountSql(executableJdbcReadQuery.getSql()) : null; - var countWithEventTimesSql = - paged ? buildCountWithEventTimesSql(tableFunction, executableJdbcReadQuery.getSql()) : null; + var pagination = + paged + ? PaginationType.OFFSET_PAGE_INFO + : hasLimitOrOffset ? PaginationType.LIMIT_AND_OFFSET : PaginationType.NONE; + var eventTimesSql = + paged ? buildEventTimesSql(tableFunction, executableJdbcReadQuery.getSql()) : null; + if (eventTimesSql != null) { + pagedRowtimeTables.add(tableFunction.getBaseTable()); + } queryBase = new SqlQuery( executableJdbcReadQuery.getSql(), parameters, - hasLimitOrOffset ? PaginationType.LIMIT_AND_OFFSET : PaginationType.NONE, + pagination, executableJdbcReadQuery.getCacheDuration().toMillis(), executableJdbcReadQuery.getDatabase(), - countSql, - countWithEventTimesSql); + eventTimesSql); var coordsBuilder = ArgumentLookupQueryCoords.builder() .parentType(parentType.getName()) @@ -192,25 +203,19 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } - /** Builds the companion COUNT(*) query for a paginated result. */ - private static String buildCountSql(String baseSql) { - return "SELECT COUNT(*) AS \"total_records\" FROM (" + baseSql + ") x"; - } - /** - * Variant of the count query that also computes MIN/MAX over the designated rowtime column for + * Builds the companion aggregate query computing MIN/MAX over the designated rowtime column for * {@code firstEventTime}/{@code lastEventTime}. Returns null when the result has no rowtime. The * rowtime column name is the same identifier as in the base query's output. */ - private static String buildCountWithEventTimesSql( - SqrlTableFunction tableFunction, String baseSql) { + private static String buildEventTimesSql(SqrlTableFunction tableFunction, String baseSql) { return tableFunction .getRowTime() .map(tableFunction::getField) .map(RelDataTypeField::getName) .map( col -> - "SELECT COUNT(*) AS \"total_records\", MIN(\"" + "SELECT MIN(\"" + col + "\") AS \"first_event_time\", MAX(\"" + col diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index 728f2e3a6c..30bd38c454 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -51,10 +51,8 @@ private OffsetPageInfoUtil() {} private static final Map PAGINATION_FIELDS = new LinkedHashMap<>(); static { - PAGINATION_FIELDS.put("totalRecords", "Long!"); PAGINATION_FIELDS.put("pageSize", "Int!"); PAGINATION_FIELDS.put("currentPage", "Int!"); - PAGINATION_FIELDS.put("totalPages", "Int!"); PAGINATION_FIELDS.put("hasNextPage", "Boolean!"); PAGINATION_FIELDS.put("hasPreviousPage", "Boolean!"); PAGINATION_FIELDS.put("nextOffset", "Int"); @@ -68,11 +66,14 @@ private OffsetPageInfoUtil() {} /** Printed type -> GraphQL type, used to build the schema type from the canonical fields. */ private static final Map GRAPHQL_TYPES = Map.of( - "Long!", GraphQLNonNull.nonNull(CustomScalars.LONG), - "Int!", GraphQLNonNull.nonNull(Scalars.GraphQLInt), - "Boolean!", GraphQLNonNull.nonNull(Scalars.GraphQLBoolean), - "Int", Scalars.GraphQLInt, - "DateTime", CustomScalars.FLEXIBLE_DATETIME); + "Int!", + GraphQLNonNull.nonNull(Scalars.GraphQLInt), + "Boolean!", + GraphQLNonNull.nonNull(Scalars.GraphQLBoolean), + "Int", + Scalars.GraphQLInt, + "DateTime", + CustomScalars.FLEXIBLE_DATETIME); /** Builds the canonical {@code OffsetPageInfo} object type for generated schemas. */ public static GraphQLObjectType createPageInfoType() { diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/PaginationType.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/PaginationType.java index 576465e48b..d5402028fb 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/PaginationType.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/PaginationType.java @@ -17,5 +17,9 @@ public enum PaginationType { NONE, - LIMIT_AND_OFFSET; + LIMIT_AND_OFFSET, + /** + * Like {@link #LIMIT_AND_OFFSET}, but the result is wrapped in an {@code OffsetPageInfo} page. + */ + OFFSET_PAGE_INFO; } diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/GraphQLEngineBuilder.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/GraphQLEngineBuilder.java index cc0dd2a96c..aee3a0a23b 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/GraphQLEngineBuilder.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/GraphQLEngineBuilder.java @@ -203,6 +203,9 @@ public ResolvedQuery visitSqlQuery(SqlQuery query, ServerContext context) { case NONE: break; case LIMIT_AND_OFFSET: + case OFFSET_PAGE_INFO: + // OFFSET_PAGE_INFO shares the limit/offset mechanics; it only adds a page wrapper computed + // at execution time. // special case where database doesn't support binding for limit/offset => need to create // query dynamically and not prepare if (!query.getDatabase().supportsLimitOffsetBinding) { diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java index b0c86faf3e..ad6954d140 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java @@ -300,19 +300,13 @@ public static class SqlQuery implements QueryBase { DatabaseType database; /** - * Companion COUNT(*) query producing pagination metadata. When non-null, the query returns a - * page wrapper ({@code {results, pagination}}) rather than a bare list. Only executed when the - * request selects pagination fields that require it. + * Companion aggregate query computing MIN/MAX over the rowtime column for {@code + * firstEventTime}/{@code lastEventTime}. Only relevant when {@link #pagination} is {@link + * PaginationType#OFFSET_PAGE_INFO}, and only executed when the request selects an event-time + * field. Null when the result has no rowtime column. */ @JsonInclude(JsonInclude.Include.NON_NULL) - String countSql; - - /** - * Variant of {@link #countSql} that additionally computes MIN/MAX over the rowtime column for - * {@code firstEventTime}/{@code lastEventTime}. Null when the result has no rowtime. - */ - @JsonInclude(JsonInclude.Include.NON_NULL) - String countWithEventTimesSql; + String eventTimesSql; @Override public R accept(QueryBaseVisitor visitor, C context) { @@ -320,14 +314,7 @@ public R accept(QueryBaseVisitor visitor, C context) { } public SqlQuery updateSql(String newSql) { - return new SqlQuery( - newSql, - parameters, - pagination, - cacheDurationMs, - database, - countSql, - countWithEventTimesSql); + return new SqlQuery(newSql, parameters, pagination, cacheDurationMs, database, eventTimesSql); } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index 5c9b92bf80..c4e80618b7 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -18,10 +18,12 @@ import static com.datasqrl.server.jdbc.SchemaConstants.LIMIT; import static com.datasqrl.server.jdbc.SchemaConstants.OFFSET; +import com.datasqrl.server.PaginationType; import com.datasqrl.server.VertxServerContext; import com.datasqrl.server.graphql.RootGraphQLModel; import com.datasqrl.server.graphql.RootGraphQLModel.Argument; import com.datasqrl.server.graphql.RootGraphQLModel.ResolvedSqlQuery; +import com.datasqrl.server.graphql.RootGraphQLModel.SqlQuery; import graphql.schema.DataFetchingEnvironment; import graphql.schema.GraphQLList; import graphql.schema.GraphQLNonNull; @@ -65,137 +67,136 @@ public CompletableFuture runQuery(ResolvedSqlQuery resolvedQuery, boolea (paramObj, throwable) -> { if (throwable != null) { cf.completeExceptionally(throwable); + } else if (resolvedQuery.getQuery().getPagination() + == PaginationType.OFFSET_PAGE_INFO) { + runPaginatedQuery(resolvedQuery, paramObj); } else { - runQueryInternal(resolvedQuery, isList, paramObj); + runPlainQuery(resolvedQuery, isList, paramObj); } }); return cf; } - private CompletableFuture runQueryInternal( + /** Executes a bare (non-paged) query, applying limit/offset when the query declares them. */ + private void runPlainQuery( ResolvedSqlQuery resolvedQuery, boolean isList, List paramObj) { - - var preparedQueryContainer = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedQueryContainer(); var query = resolvedQuery.getQuery(); - var unpreparedSqlQuery = query.getSql(); - var database = query.getDatabase(); - var paged = query.getCountSql() != null; - - // Pagination metadata is computed lazily from the selection set: the aggregate query only - // runs when totals or event times are selected, and hasNextPage without totals is derived - // by fetching one extra row instead. - PageFieldNames fieldNames = null; - String aggregateSql = null; - var needNextWithoutAggregate = false; - if (paged) { - fieldNames = pageFieldNames(environment.getFieldType()); - var pag = fieldNames.paginationField(); - var selection = environment.getSelectionSet(); - var needEventTimes = selection.containsAnyOf(pag + "/firstEventTime", pag + "/lastEventTime"); - var needTotals = selection.containsAnyOf(pag + "/totalRecords", pag + "/totalPages"); - if (needEventTimes && query.getCountWithEventTimesSql() != null) { - aggregateSql = query.getCountWithEventTimesSql(); - } else if (needTotals) { - aggregateSql = query.getCountSql(); - } - needNextWithoutAggregate = - aggregateSql == null - && selection.containsAnyOf(pag + "/hasNextPage", pag + "/nextOffset"); + var sql = query.getSql(); + if (query.getPagination() == PaginationType.LIMIT_AND_OFFSET) { + var limit = Optional.ofNullable(environment.getArgument(LIMIT)); + var offset = Optional.ofNullable(environment.getArgument(OFFSET)); + sql = + applyLimitOffset( + query, + sql, + paramObj, + limit.orElse(Integer.MAX_VALUE), + offset.orElse(0), + limit.isPresent()); } - // The aggregate query binds the base parameters only, without the runtime limit/offset. - // Tuple.from snapshots the list here (before limit/offset are appended below) and, unlike - // List.copyOf, tolerates null bind values (SQL NULL). - var countParams = aggregateSql != null ? Tuple.from(paramObj) : null; + execute(resolvedQuery, sql, paramObj) + .map(r -> resultMapper(r, isList)) + .onSuccess(cf::complete) + .onFailure( + f -> { + f.printStackTrace(); + cf.completeExceptionally(f); + }); + } - int limitValue = Integer.MAX_VALUE; - int offsetValue = 0; - var fetchExtraRow = false; - switch (query.getPagination()) { - case NONE: - break; - case LIMIT_AND_OFFSET: - var limit = Optional.ofNullable(environment.getArgument(LIMIT)); - var offset = Optional.ofNullable(environment.getArgument(OFFSET)); - limitValue = limit.orElse(Integer.MAX_VALUE); - offsetValue = offset.orElse(0); + /** + * Executes an {@link PaginationType#OFFSET_PAGE_INFO} query: the page data query always runs, + * while the pagination metadata is computed lazily from the selection set. The event-time + * aggregate runs only when an event-time field is selected; {@code hasNextPage}/{@code + * nextOffset} are answered by fetching one extra row instead of any aggregate. + */ + private void runPaginatedQuery(ResolvedSqlQuery resolvedQuery, List paramObj) { + var query = resolvedQuery.getQuery(); + var fieldNames = pageFieldNames(environment.getFieldType()); + var pag = fieldNames.paginationField(); + var selection = environment.getSelectionSet(); - // without a limit the page contains every remaining row, so there is no next page and - // no extra row to fetch - fetchExtraRow = needNextWithoutAggregate && limitValue != Integer.MAX_VALUE; - var fetchLimit = fetchExtraRow ? limitValue + 1 : limitValue; + var needEventTimes = selection.containsAnyOf(pag + "/firstEventTime", pag + "/lastEventTime"); + var eventTimesSql = needEventTimes ? query.getEventTimesSql() : null; - // special case where database doesn't support binding for limit/offset => need - // to execute dynamically - if (!query.getDatabase().supportsLimitOffsetBinding) { - assert preparedQueryContainer == null; - unpreparedSqlQuery = - AbstractQueryExecutionContext.addLimitOffsetToQuery( - unpreparedSqlQuery, - limit.isPresent() ? String.valueOf(fetchLimit) : "ALL", - String.valueOf(offsetValue)); - } else { - paramObj.add(fetchLimit); - paramObj.add(offsetValue); - } - break; - default: - throw new UnsupportedOperationException("Unsupported pagination: " + query.getPagination()); - } + var limit = Optional.ofNullable(environment.getArgument(LIMIT)); + var offset = Optional.ofNullable(environment.getArgument(OFFSET)); + var limitValue = limit.orElse(Integer.MAX_VALUE); + var offsetValue = offset.orElse(0); - // execute the preparedQuery with the arguments extracted above - Future> future; - var params = Tuple.from(paramObj); + // hasNextPage/nextOffset are derived by fetching one extra row. Without a limit the page holds + // every remaining row, so there is no next page and no extra row to fetch. + var deriveNext = selection.containsAnyOf(pag + "/hasNextPage", pag + "/nextOffset"); + var fetchExtraRow = deriveNext && limitValue != Integer.MAX_VALUE; + var fetchLimit = fetchExtraRow ? limitValue + 1 : limitValue; - if (preparedQueryContainer == null) { - future = serverContext.getSqlClient().execute(database, unpreparedSqlQuery, params); - } else { - var preparedQuery = preparedQueryContainer.preparedQuery(); - future = serverContext.getSqlClient().execute(preparedQuery, params); - } + // The aggregate binds the base parameters only. Tuple.from snapshots the list here (before + // limit/offset are appended below) and, unlike List.copyOf, tolerates null bind values. + var aggregateParams = eventTimesSql != null ? Tuple.from(paramObj) : null; - if (paged) { - Future> aggregateFuture = - aggregateSql == null - ? Future.succeededFuture(null) - : serverContext.getSqlClient().execute(database, aggregateSql, countParams); - var dataFuture = future; - var pageNames = fieldNames; - var effectiveLimit = limitValue; - var effectiveOffset = offsetValue; - var extraRowFetched = fetchExtraRow; - var deriveNextFromResults = needNextWithoutAggregate; - Future.all(dataFuture, aggregateFuture) - .map( - c -> - pagedResultMapper( - dataFuture.result(), - aggregateFuture.result(), - pageNames, - effectiveLimit, - effectiveOffset, - extraRowFetched, - deriveNextFromResults)) - .onSuccess(cf::complete) - .onFailure( - f -> { - f.printStackTrace(); - cf.completeExceptionally(f); - }); - return cf; - } + var sql = + applyLimitOffset( + query, query.getSql(), paramObj, fetchLimit, offsetValue, limit.isPresent()); - // map the resultSet to json for GraphQL response - future - .map(r -> resultMapper(r, isList)) + var dataFuture = execute(resolvedQuery, sql, paramObj); + Future> aggregateFuture = + eventTimesSql == null + ? Future.succeededFuture(null) + : serverContext + .getSqlClient() + .execute(query.getDatabase(), eventTimesSql, aggregateParams); + + Future.all(dataFuture, aggregateFuture) + .map( + c -> + pagedResultMapper( + dataFuture.result(), + aggregateFuture.result(), + fieldNames, + limitValue, + offsetValue, + fetchExtraRow, + deriveNext)) .onSuccess(cf::complete) .onFailure( f -> { f.printStackTrace(); cf.completeExceptionally(f); }); - return cf; + } + + /** + * Binds limit/offset as parameters (databases that support it) or rewrites the SQL text + * (databases that don't, e.g. Snowflake). Returns the SQL to execute. + */ + private static String applyLimitOffset( + SqlQuery query, + String sql, + List paramObj, + int limit, + int offset, + boolean limitPresent) { + if (!query.getDatabase().supportsLimitOffsetBinding) { + return AbstractQueryExecutionContext.addLimitOffsetToQuery( + sql, limitPresent ? String.valueOf(limit) : "ALL", String.valueOf(offset)); + } + paramObj.add(limit); + paramObj.add(offset); + return sql; + } + + private Future> execute( + ResolvedSqlQuery resolvedQuery, String sql, List paramObj) { + var container = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedQueryContainer(); + var params = Tuple.from(paramObj); + if (container == null) { + return serverContext + .getSqlClient() + .execute(resolvedQuery.getQuery().getDatabase(), sql, params); + } + return serverContext.getSqlClient().execute(container.preparedQuery(), params); } private Object resultMapper(RowSet r, boolean isList) { @@ -206,7 +207,7 @@ private Object resultMapper(RowSet r, boolean isList) { private Object pagedResultMapper( RowSet dataRows, - RowSet aggregateRows, + RowSet eventTimeRows, PageFieldNames fieldNames, int limit, int offset, @@ -224,21 +225,17 @@ private Object pagedResultMapper( hasNextPage = false; // no limit given: the page contains every remaining row } - Long totalRecords = null; Object firstEventTime = null; Object lastEventTime = null; - if (aggregateRows != null) { - var aggregateIterator = aggregateRows.iterator(); - var aggregateJson = - aggregateIterator.hasNext() ? aggregateIterator.next().toJson() : new JsonObject(); - totalRecords = aggregateJson.getLong("total_records", 0L); - firstEventTime = aggregateJson.getValue("first_event_time"); - lastEventTime = aggregateJson.getValue("last_event_time"); + if (eventTimeRows != null) { + var it = eventTimeRows.iterator(); + var eventTimeJson = it.hasNext() ? it.next().toJson() : new JsonObject(); + firstEventTime = eventTimeJson.getValue("first_event_time"); + lastEventTime = eventTimeJson.getValue("last_event_time"); } var pagination = - buildPaginationMetadata( - totalRecords, hasNextPage, limit, offset, firstEventTime, lastEventTime); + buildPaginationMetadata(hasNextPage, limit, offset, firstEventTime, lastEventTime); // An absent limit means "return everything" (limit == Integer.MAX_VALUE); report the actual // number of rows on this page as pageSize rather than leaking the sentinel. if (limit == Integer.MAX_VALUE) { @@ -274,17 +271,12 @@ private static PageFieldNames pageFieldNames(GraphQLOutputType fieldType) { private record PageFieldNames(String resultsField, String paginationField) {} /** - * Builds the pagination metadata object. {@code totalRecords} and {@code hasNextPage} are null - * when the request did not select fields requiring them; the corresponding fields are then left - * out (GraphQL never reads unselected fields). + * Builds the pagination metadata object. {@code hasNextPage} is null when the request did not + * select fields requiring it; the corresponding fields are then left out (GraphQL never reads + * unselected fields). */ static JsonObject buildPaginationMetadata( - Long totalRecords, - Boolean hasNextPage, - int limit, - int offset, - Object firstEventTime, - Object lastEventTime) { + Boolean hasNextPage, int limit, int offset, Object firstEventTime, Object lastEventTime) { boolean hasPreviousPage = offset > 0; var pagination = new JsonObject() @@ -296,12 +288,6 @@ static JsonObject buildPaginationMetadata( .put("firstEventTime", firstEventTime) .put("lastEventTime", lastEventTime); - if (totalRecords != null) { - pagination - .put("totalRecords", totalRecords) - .put("totalPages", limit == 0 ? 0 : (int) Math.ceil((double) totalRecords / limit)); - hasNextPage = (long) offset + limit < totalRecords; - } if (hasNextPage != null) { pagination .put("hasNextPage", hasNextPage) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java index 8f0e1888eb..56a1d86e4c 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -59,20 +59,17 @@ import org.testcontainers.utility.DockerImageName; /** - * Proves that pagination metadata is computed lazily from the selection set: the aggregate query - * only runs when totals or event times are selected, event times pick the MIN/MAX variant, and - * {@code hasNextPage} alone is answered by fetching LIMIT+1 rows instead of any count query. + * Proves that pagination metadata is computed lazily from the selection set: the MIN/MAX aggregate + * query only runs when event times are selected, and {@code hasNextPage} alone is answered by + * fetching LIMIT+1 rows instead of any aggregate. */ @ExtendWith(VertxExtension.class) @Testcontainers class PagedQueryIT { private static final String BASE_SQL = "SELECT customerid, ts FROM customer ORDER BY customerid"; - private static final String COUNT_SQL = - "SELECT COUNT(*) AS \"total_records\" FROM (" + BASE_SQL + ") x"; - private static final String COUNT_WITH_EVENT_TIMES_SQL = - "SELECT COUNT(*) AS \"total_records\", MIN(\"ts\") AS \"first_event_time\"," - + " MAX(\"ts\") AS \"last_event_time\" FROM (" + private static final String EVENT_TIMES_SQL = + "SELECT MIN(\"ts\") AS \"first_event_time\", MAX(\"ts\") AS \"last_event_time\" FROM (" + BASE_SQL + ") x"; @@ -187,36 +184,15 @@ void givenHasNextPageSelectedOnLastPage_whenQuery_thenNoNextPage() { } @Test - void givenTotalsSelected_whenQuery_thenPlainCountRunsWithoutExtraRow() { - var customers = - execute( - "{ customers(limit: 2, offset: 0) { results { customerid }" - + " pagination { totalRecords totalPages hasNextPage } } }"); - - assertThat(recordingClient.executed).hasSize(2); - assertThat(sqlOf(recordingClient.executed)).contains(COUNT_SQL); - // hasNextPage is derived from the count, so the data query does not fetch an extra row - var dataStatement = - recordingClient.executed.stream().filter(s -> !s.sql().contains("COUNT")).findFirst(); - assertThat(dataStatement).isPresent(); - assertThat(dataStatement.get().params().getInteger(0)).isEqualTo(2); - assertThat(pagination(customers)) - .containsEntry("totalRecords", 5L) - .containsEntry("totalPages", 3) - .containsEntry("hasNextPage", true); - } - - @Test - void givenEventTimesSelected_whenQuery_thenMinMaxVariantRuns() { + void givenEventTimesSelected_whenQuery_thenMinMaxAggregateRuns() { var customers = execute( "{ customers(limit: 2, offset: 2) { results { customerid }" - + " pagination { totalRecords firstEventTime lastEventTime } } }"); + + " pagination { firstEventTime lastEventTime } } }"); assertThat(recordingClient.executed).hasSize(2); - assertThat(sqlOf(recordingClient.executed)).contains(COUNT_WITH_EVENT_TIMES_SQL); + assertThat(sqlOf(recordingClient.executed)).contains(EVENT_TIMES_SQL); var pagination = pagination(customers); - assertThat(pagination).containsEntry("totalRecords", 5L); // MIN/MAX cover the whole result, not just the requested page assertThat(String.valueOf(pagination.get("firstEventTime"))).startsWith("2024-01-01"); assertThat(String.valueOf(pagination.get("lastEventTime"))).startsWith("2024-01-05"); @@ -227,29 +203,26 @@ void givenNoLimitArgument_whenQuery_thenPageSizeReportsRowCountNotSentinel() { var customers = execute( "{ customers: customersUnbounded { results { customerid }" - + " pagination { pageSize totalRecords hasNextPage } } }"); + + " pagination { pageSize hasNextPage } } }"); assertThat(results(customers)).hasSize(5); // an absent limit fetches every row; pageSize reflects the rows returned, not Integer.MAX_VALUE assertThat(pagination(customers)) .containsEntry("pageSize", 5) - .containsEntry("totalRecords", 5L) .containsEntry("hasNextPage", false); } @Test - void givenEventTimesSelectedButNoRowtime_whenQuery_thenPlainCountRunsAndEventTimesAreNull() { + void givenEventTimesSelectedButNoRowtime_whenQuery_thenNoAggregateRunsAndEventTimesAreNull() { var customers = execute( "{ customers: customersNoRowtime(limit: 2) { results { customerid }" - + " pagination { totalRecords firstEventTime lastEventTime } } }"); + + " pagination { firstEventTime lastEventTime } } }"); - assertThat(recordingClient.executed).hasSize(2); - // no rowtime => the MIN/MAX variant is absent; the plain count runs and event times stay null - assertThat(sqlOf(recordingClient.executed)).contains(COUNT_SQL); - assertThat(sqlOf(recordingClient.executed)).doesNotContain(COUNT_WITH_EVENT_TIMES_SQL); + // no rowtime => no aggregate query exists, so only the data query runs and event times stay + // null + assertThat(recordingClient.executed).hasSize(1); var pagination = pagination(customers); - assertThat(pagination).containsEntry("totalRecords", 5L); assertThat(pagination.get("firstEventTime")).isNull(); assertThat(pagination.get("lastEventTime")).isNull(); } @@ -283,7 +256,6 @@ private RootGraphQLModel getPagedModel() { .schema( """ scalar DateTime - scalar Long type Query { customers(limit: Int = 10, offset: Int = 0): CustomerPage! customersUnbounded(limit: Int, offset: Int = 0): CustomerPage! @@ -297,10 +269,8 @@ private RootGraphQLModel getPagedModel() { pagination: OffsetPageInfo } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -320,11 +290,10 @@ private RootGraphQLModel getPagedModel() { new SqlQuery( BASE_SQL, List.of(), - PaginationType.LIMIT_AND_OFFSET, + PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - COUNT_SQL, - COUNT_WITH_EVENT_TIMES_SQL)) + EVENT_TIMES_SQL)) .build()) .build()) .query( @@ -337,11 +306,10 @@ private RootGraphQLModel getPagedModel() { new SqlQuery( BASE_SQL, List.of(), - PaginationType.LIMIT_AND_OFFSET, + PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - COUNT_SQL, - COUNT_WITH_EVENT_TIMES_SQL)) + EVENT_TIMES_SQL)) .build()) .build()) .query( @@ -354,10 +322,9 @@ private RootGraphQLModel getPagedModel() { new SqlQuery( BASE_SQL, List.of(), - PaginationType.LIMIT_AND_OFFSET, + PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - COUNT_SQL, null)) .build()) .build()) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java index 6ffd145af3..63d24ce7b1 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java @@ -172,7 +172,6 @@ private RootGraphQLModel getCustomerModel() { PaginationType.NONE, 0, DatabaseType.POSTGRES, - null, null)) .build()) .build()) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java index 699e9993cf..c51fd13696 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java @@ -22,25 +22,11 @@ class PaginationMetadataTest { @Test - void givenEmptyResult_whenBuildMetadata_thenSinglePageNoRecords() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(0L, null, 10, 0, null, null); + void givenFirstPageWithNext_whenBuildMetadata_thenHasNextNoPrevious() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(true, 10, 0, null, null); - assertThat(json.getLong("totalRecords")).isZero(); assertThat(json.getInteger("pageSize")).isEqualTo(10); assertThat(json.getInteger("currentPage")).isEqualTo(1); - assertThat(json.getInteger("totalPages")).isZero(); - assertThat(json.getBoolean("hasNextPage")).isFalse(); - assertThat(json.getBoolean("hasPreviousPage")).isFalse(); - assertThat(json.getInteger("nextOffset")).isNull(); - assertThat(json.getInteger("prevOffset")).isNull(); - } - - @Test - void givenFirstPage_whenBuildMetadata_thenHasNextNoPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 0, null, null); - - assertThat(json.getInteger("currentPage")).isEqualTo(1); - assertThat(json.getInteger("totalPages")).isEqualTo(3); assertThat(json.getBoolean("hasNextPage")).isTrue(); assertThat(json.getBoolean("hasPreviousPage")).isFalse(); assertThat(json.getInteger("nextOffset")).isEqualTo(10); @@ -48,8 +34,8 @@ void givenFirstPage_whenBuildMetadata_thenHasNextNoPrevious() { } @Test - void givenMiddlePage_whenBuildMetadata_thenHasBothNeighbours() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 10, null, null); + void givenMiddlePageWithNext_whenBuildMetadata_thenHasBothNeighbours() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(true, 10, 10, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(2); assertThat(json.getBoolean("hasNextPage")).isTrue(); @@ -59,30 +45,20 @@ void givenMiddlePage_whenBuildMetadata_thenHasBothNeighbours() { } @Test - void givenLastPartialPage_whenBuildMetadata_thenNoNextHasPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 20, null, null); + void givenLastPage_whenBuildMetadata_thenNoNextHasPrevious() { + var json = VertxQueryExecutionContext.buildPaginationMetadata(false, 10, 20, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(3); assertThat(json.getBoolean("hasNextPage")).isFalse(); - assertThat(json.getBoolean("hasPreviousPage")).isTrue(); assertThat(json.getInteger("nextOffset")).isNull(); - assertThat(json.getInteger("prevOffset")).isEqualTo(10); - } - - @Test - void givenOffsetBeyondTotal_whenBuildMetadata_thenNoNextPage() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 10, 30, null, null); - - assertThat(json.getBoolean("hasNextPage")).isFalse(); assertThat(json.getBoolean("hasPreviousPage")).isTrue(); - assertThat(json.getInteger("prevOffset")).isEqualTo(20); + assertThat(json.getInteger("prevOffset")).isEqualTo(10); } @Test void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, null, 0, 0, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(null, 0, 0, null, null); - assertThat(json.getInteger("totalPages")).isZero(); assertThat(json.getInteger("currentPage")).isEqualTo(1); } @@ -90,39 +66,19 @@ void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { void givenEventTimes_whenBuildMetadata_thenPassedThrough() { var json = VertxQueryExecutionContext.buildPaginationMetadata( - 5L, null, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); + null, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); assertThat(json.getString("firstEventTime")).isEqualTo("2024-01-01T00:00:00Z"); assertThat(json.getString("lastEventTime")).isEqualTo("2024-01-02T00:00:00Z"); } - @Test - void givenNoTotalsQueried_whenBuildMetadata_thenTotalsOmitted() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(null, true, 10, 10, null, null); - - assertThat(json.containsKey("totalRecords")).isFalse(); - assertThat(json.containsKey("totalPages")).isFalse(); - assertThat(json.getBoolean("hasNextPage")).isTrue(); - assertThat(json.getInteger("nextOffset")).isEqualTo(20); - assertThat(json.getBoolean("hasPreviousPage")).isTrue(); - assertThat(json.getInteger("prevOffset")).isZero(); - } - @Test void givenNoNextPageInfoQueried_whenBuildMetadata_thenNextFieldsOmitted() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(null, null, 10, 0, null, null); + var json = VertxQueryExecutionContext.buildPaginationMetadata(null, 10, 0, null, null); assertThat(json.containsKey("hasNextPage")).isFalse(); assertThat(json.containsKey("nextOffset")).isFalse(); assertThat(json.getInteger("pageSize")).isEqualTo(10); assertThat(json.getBoolean("hasPreviousPage")).isFalse(); } - - @Test - void givenTotalsQueried_whenBuildMetadata_thenHasNextDerivedFromTotals() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(25L, false, 10, 0, null, null); - - assertThat(json.getBoolean("hasNextPage")).isTrue(); - assertThat(json.getInteger("nextOffset")).isEqualTo(10); - } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls index 644c750d93..9af1217a3a 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls @@ -17,10 +17,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls index 5645b9cc2d..bc41941f10 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls @@ -18,10 +18,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls index 8b711ba362..70a840a44a 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls @@ -17,10 +17,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls index c80fc870f8..2119ac727f 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls @@ -4,10 +4,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt index 833ec77664..3e2bcc6bad 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt @@ -20,7 +20,7 @@ CustomerTimeWindow := SELECT ^ [FATAL] Paginated query [CustomerByTime2] must declare both 'limit' and 'offset' arguments -in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [33:5]: +in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [31:5]: type Query { CustomerByTime2: CustomerPage! diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt index 74602170fe..f1e03f13fe 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt @@ -21,10 +21,8 @@ CustomerTimeWindow := SELECT [FATAL] Paginated results require the OffsetPageInfo type to be declared in the schema: type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt index 6d44cb60fb..a7a8c6905f 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt @@ -21,10 +21,8 @@ CustomerTimeWindow := SELECT [FATAL] User-defined OffsetPageInfo does not match the expected definition: type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt index 8473bfa7a8..9011055339 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt @@ -289,10 +289,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,10 +374,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -588,10 +584,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -611,10 +605,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index 69faa45aa6..9869a474e4 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -289,10 +289,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,10 +374,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -588,10 +584,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -611,10 +605,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -1220,10 +1212,10 @@ CREATE TABLE `Customer` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `Customer__schema`; CREATE TEMPORARY TABLE `ExternalOrders__schema` ( @@ -1240,10 +1232,10 @@ CREATE TABLE `ExternalOrders` ( WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `ExternalOrders__schema`; CREATE TEMPORARY TABLE `_Customer__schema` ( @@ -1261,10 +1253,10 @@ CREATE TABLE `_Customer` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Customer__schema`; CREATE TEMPORARY TABLE `_Orders__schema` ( @@ -1281,10 +1273,10 @@ CREATE TABLE `_Orders` ( WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Orders__schema`; CREATE TEMPORARY TABLE `_Product__schema` ( @@ -1302,10 +1294,10 @@ CREATE TABLE `_Product` ( WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Product__schema`; CREATE VIEW `CustomerByTime2` @@ -1807,6 +1799,7 @@ CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BI CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); +CREATE INDEX IF NOT EXISTS "CustomerByTime2_btree_c4" ON "CustomerByTime2" USING btree ("timestamp"); >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * @@ -1837,11 +1830,10 @@ FROM "ExternalOrders" AS "ExternalOrders0" "type" : "SqlQuery", "sql" : "SELECT *\nFROM \"CustomerByTime2\"", "parameters" : [ ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"CustomerByTime2\") x", - "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" } } } @@ -1867,7 +1859,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" }, "format" : "JSON", "apiQuery" : { - "query" : "query CustomerByTime2($limit: Int = 10, $offset: Int = 0) {\nCustomerByTime2(limit: $limit, offset: $offset) {\nresults {\ncustomerid\nemail\nname\nlastUpdated\ntimestamp\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query CustomerByTime2($limit: Int = 10, $offset: Int = 0) {\nCustomerByTime2(limit: $limit, offset: $offset) {\nresults {\ncustomerid\nemail\nname\nlastUpdated\ntimestamp\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "CustomerByTime2", "operationType" : "QUERY" }, @@ -1878,7 +1870,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 0aaf2c981d..678bff3af1 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -289,10 +289,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,10 +374,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -588,10 +584,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -611,10 +605,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -1220,10 +1212,10 @@ CREATE TABLE `Customer` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `Customer__schema`; CREATE TEMPORARY TABLE `ExternalOrders__schema` ( @@ -1240,10 +1232,10 @@ CREATE TABLE `ExternalOrders` ( WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `ExternalOrders__schema`; CREATE TEMPORARY TABLE `_Customer__schema` ( @@ -1261,10 +1253,10 @@ CREATE TABLE `_Customer` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Customer__schema`; CREATE TEMPORARY TABLE `_Orders__schema` ( @@ -1281,10 +1273,10 @@ CREATE TABLE `_Orders` ( WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Orders__schema`; CREATE TEMPORARY TABLE `_Product__schema` ( @@ -1302,10 +1294,10 @@ CREATE TABLE `_Product` ( WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = 'file:/mock', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `_Product__schema`; CREATE VIEW `CustomerByTime2` @@ -1807,6 +1799,8 @@ CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BI CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); +CREATE INDEX IF NOT EXISTS "Customer_btree_c4" ON "Customer" USING btree ("timestamp"); +CREATE INDEX IF NOT EXISTS "SelectCustomers_btree_c4" ON "SelectCustomers" USING btree ("timestamp"); >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * @@ -1856,11 +1850,10 @@ FROM "ExternalOrders" AS "ExternalOrders0" "sqlType" : "INTEGER" } ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x", - "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" } } }, @@ -1888,11 +1881,10 @@ FROM "ExternalOrders" AS "ExternalOrders0" "key" : "customerid" } ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x", - "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" } } } @@ -1902,7 +1894,7 @@ FROM "ExternalOrders" AS "ExternalOrders0" "operations" : [ ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt index 9b0a373932..e893d5c43a 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt @@ -289,10 +289,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,10 +374,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -588,10 +584,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -611,10 +605,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt index ba11510792..b9a653163b 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt @@ -289,10 +289,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,10 +374,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -588,10 +584,8 @@ type CustomerPage { } type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -611,10 +605,8 @@ scalar DateTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt index 9d415f0b34..6f61374a5b 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt @@ -26,10 +26,8 @@ scalar LocalTime scalar Long type OffsetPageInfo { - totalRecords: Long! pageSize: Int! currentPage: Int! - totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -184,10 +182,10 @@ CREATE TABLE `Click` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = '${DATA_PATH}/click.jsonl', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `Click__schema`; CREATE VIEW `Trending` @@ -288,6 +286,9 @@ CREATE TABLE IF NOT EXISTS "Recommendation" ("url" TEXT NOT NULL, "rec" TEXT NOT CREATE TABLE IF NOT EXISTS "Trending" ("url" TEXT NOT NULL, "total" BIGINT NOT NULL, PRIMARY KEY ("url")); CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" TEXT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); +CREATE INDEX IF NOT EXISTS "Click_btree_c1" ON "Click" USING btree ("timestamp"); +CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("timestamp"); + >>>vertx.json { "models" : { @@ -312,11 +313,10 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T "type" : "SqlQuery", "sql" : "SELECT *\nFROM \"Click\"", "parameters" : [ ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Click\") x", - "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x" } } }, @@ -349,10 +349,9 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T "sqlType" : "VARCHAR" } ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, - "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"rec\", \"frequency\"\n FROM \"Recommendation\"\n ORDER BY \"url\" NULLS FIRST, \"frequency\" DESC NULLS LAST) AS \"t\"\nWHERE \"url\" = $1) x" + "database" : "POSTGRES" } } }, @@ -375,10 +374,9 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T "type" : "SqlQuery", "sql" : "SELECT *\nFROM (SELECT \"url\", \"total\"\n FROM \"Trending\"\n ORDER BY \"total\" DESC NULLS LAST, \"url\" NULLS FIRST) AS \"t\"", "parameters" : [ ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, - "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"total\"\n FROM \"Trending\"\n ORDER BY \"total\" DESC NULLS LAST, \"url\" NULLS FIRST) AS \"t\") x" + "database" : "POSTGRES" } } }, @@ -401,11 +399,10 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T "type" : "SqlQuery", "sql" : "SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\"", "parameters" : [ ], - "pagination" : "LIMIT_AND_OFFSET", + "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x", - "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" } } } @@ -431,7 +428,7 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T }, "format" : "JSON", "apiQuery" : { - "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\nurl\ntimestamp\nuserid\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\nurl\ntimestamp\nuserid\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Click", "operationType" : "QUERY" }, @@ -464,7 +461,7 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T }, "format" : "JSON", "apiQuery" : { - "query" : "query Recommendation($url: String!, $limit: Int = 10, $offset: Int = 0) {\nRecommendation(url: $url, limit: $limit, offset: $offset) {\nresults {\nurl\nrec\nfrequency\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Recommendation($url: String!, $limit: Int = 10, $offset: Int = 0) {\nRecommendation(url: $url, limit: $limit, offset: $offset) {\nresults {\nurl\nrec\nfrequency\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Recommendation", "operationType" : "QUERY" }, @@ -491,7 +488,7 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T }, "format" : "JSON", "apiQuery" : { - "query" : "query Trending($limit: Int = 10, $offset: Int = 0) {\nTrending(limit: $limit, offset: $offset) {\nresults {\nurl\ntotal\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Trending($limit: Int = 10, $offset: Int = 0) {\nTrending(limit: $limit, offset: $offset) {\nresults {\nurl\ntotal\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Trending", "operationType" : "QUERY" }, @@ -517,7 +514,7 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T }, "format" : "JSON", "apiQuery" : { - "query" : "query VisitAfter($limit: Int = 10, $offset: Int = 0) {\nVisitAfter(limit: $limit, offset: $offset) {\nresults {\nbeforeURL\nafterURL\ntimestamp\n}\npagination {\ntotalRecords\npageSize\ncurrentPage\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query VisitAfter($limit: Int = 10, $offset: Int = 0) {\nVisitAfter(limit: $limit, offset: $offset) {\nresults {\nbeforeURL\nafterURL\ntimestamp\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "VisitAfter", "operationType" : "QUERY" }, @@ -528,7 +525,7 @@ CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" T ], "schema" : { "type" : "string", - "schema" : "type Click {\n url: String!\n timestamp: DateTime!\n userid: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n totalRecords: Long!\n pageSize: Int!\n currentPage: Int!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" + "schema" : "type Click {\n url: String!\n timestamp: DateTime!\n userid: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" } } } From 3a461bb45b881fd199d5031a0117c19983373b11 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Fri, 10 Jul 2026 11:13:33 -0300 Subject: [PATCH 09/16] test: Regenerate DAG writer snapshot for paginated clickstream usecase Signed-off-by: Marvin Froeder --- .../DAGWriterJsonTest/clickstream-package-paginated.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt index 22e0f47677..3fc2dab376 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt @@ -66,7 +66,7 @@ "description" : "Click" } ], "plan" : "LogicalWatermarkAssigner(rowtime=[timestamp], watermark=[-($1, 1000:INTERVAL SECOND)])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", - "sql" : "CREATE TEMPORARY TABLE `Click__schema` (\n `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL,\n `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL,\n `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL\n)\nWITH (\n 'connector' = 'datagen'\n);\nCREATE TABLE `Click` (\n PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED,\n WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND\n)\nWITH (\n 'format' = 'flexible-json',\n 'path' = '${DATA_PATH}/click.jsonl',\n 'source.monitor-interval' = '10 sec',\n 'connector' = 'filesystem'\n)\nLIKE `Click__schema`", + "sql" : "CREATE TEMPORARY TABLE `Click__schema` (\n `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL,\n `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL,\n `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL\n)\nWITH (\n 'connector' = 'datagen'\n);\nCREATE TABLE `Click` (\n PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED,\n WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND\n)\nWITH (\n 'connector' = 'filesystem',\n 'format' = 'flexible-json',\n 'path' = '${DATA_PATH}/click.jsonl',\n 'source.monitor-interval' = '10 sec'\n)\nLIKE `Click__schema`", "timestamp" : "timestamp", "schema" : [ { "name" : "url", @@ -180,10 +180,10 @@ CREATE TABLE `Click` ( WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND ) WITH ( + 'connector' = 'filesystem', 'format' = 'flexible-json', 'path' = '${DATA_PATH}/click.jsonl', - 'source.monitor-interval' = '10 sec', - 'connector' = 'filesystem' + 'source.monitor-interval' = '10 sec' ) LIKE `Click__schema`; /** Most visited pages From 10285897e0d0ec6cbd30f42beb9ce8eb445bb400 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 22 Jul 2026 13:16:02 -0300 Subject: [PATCH 10/16] refactor: Separate OFFSET_PAGE_INFO execution from LIMIT_AND_OFFSET and document pagination Signed-off-by: Marvin Froeder --- documentation/docs/configuration-default.md | 8 +- documentation/docs/configuration.md | 3 +- documentation/docs/interface.md | 47 ++++ .../datasqrl/server/GraphqlSchemaWalker.java | 4 +- .../datasqrl/server/OffsetPageInfoUtil.java | 5 +- .../server/jdbc/OffsetPageInfoQuery.java | 166 +++++++++++ .../com/datasqrl/server/jdbc/PageRequest.java | 83 ++++++ .../jdbc/VertxQueryExecutionContext.java | 266 ++++-------------- .../server/jdbc/PaginationMetadataTest.java | 14 +- 9 files changed, 375 insertions(+), 221 deletions(-) create mode 100644 sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java create mode 100644 sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/PageRequest.java diff --git a/documentation/docs/configuration-default.md b/documentation/docs/configuration-default.md index bf98faef51..9f3d6515c0 100644 --- a/documentation/docs/configuration-default.md +++ b/documentation/docs/configuration-default.md @@ -10,7 +10,7 @@ The following is the [default configuration file](https://raw.githubusercontent. "logger": "print", "compile-flink-plan": true, "extended-scalar-types": true, - "predicate-pushdown-rules": "LIMITED_TABLE_SOURCE_RULES", + "predicate-pushdown-rules": "LIMITED_RULES_NO_SOURCE", "cost-model": "DEFAULT", "explain": { "sql": false, @@ -23,7 +23,8 @@ The following is the [default configuration file](https://raw.githubusercontent. "endpoints": "FULL", "add-prefix": true, "max-result-depth": 3, - "default-limit": 10 + "default-limit": 10, + "paginated-results": false } }, "engines": { @@ -31,7 +32,8 @@ The following is the [default configuration file](https://raw.githubusercontent. "config": { "execution.runtime-mode": "STREAMING", "security.delegation.tokens.enabled": false, - "state.backend.type": "rocksdb" + "state.backend.type": "rocksdb", + "table.exec.sink.require-on-conflict": false } }, "duckdb": { diff --git a/documentation/docs/configuration.md b/documentation/docs/configuration.md index 08141d17c1..f48ba68454 100644 --- a/documentation/docs/configuration.md +++ b/documentation/docs/configuration.md @@ -162,7 +162,8 @@ Configuration options that control the compiler, such as where logging output is "endpoints": "FULL", // endpoint generation strategy ("FULL", "GRAPHQL", "OPS_ONLY") "add-prefix": true, // add an operation-type prefix to function names to ensure uniqueness "max-result-depth": 3, // maximum depth of graph traversal when generating operations from a schema - "default-limit": 10 // default query result limit + "default-limit": 10, // default query result limit + "paginated-results": false // wrap generated query results in a page with pagination metadata } } } diff --git a/documentation/docs/interface.md b/documentation/docs/interface.md index ed405b60d8..b4c31ec151 100644 --- a/documentation/docs/interface.md +++ b/documentation/docs/interface.md @@ -67,6 +67,53 @@ You can customize the GraphQL schema by: The compiler raises errors when the provided GraphQL schema is not compatible with the object-relationship model. ::: +#### Pagination + +Every generated query endpoint takes `limit` and `offset` arguments to page through the result (`limit` defaults to the configured `default-limit`). By default the endpoint returns the rows directly and the client tracks the offsets itself. + +Set `paginated-results` to `true` in the [`api` compiler configuration](configuration.md#compiler-compiler) to get pagination metadata alongside the rows. The generated schema then wraps every multi-row query result in a page type: + +```graphql +type PersonPage { + results: [Person!] + pagination: OffsetPageInfo +} + +type OffsetPageInfo { + pageSize: Int! + currentPage: Int! + hasNextPage: Boolean! + hasPreviousPage: Boolean! + nextOffset: Int + prevOffset: Int + firstEventTime: DateTime + lastEventTime: DateTime +} +``` + +A query then selects the rows and the metadata it needs: + +```graphql +query GetPeople { + Person(limit: 10, offset: 20) { + results { name email } + pagination { currentPage hasNextPage nextOffset } + } +} +``` + +`firstEventTime` and `lastEventTime` are the earliest and latest event time of the *entire* result set, not of the returned page. They are `null` when the query result has no event time (rowtime) column. + +If you provide your own GraphQL schema, pagination is opt-in per query: give the query a result type with exactly two fields — a list of the result type, and a field of type `OffsetPageInfo` — and declare `OffsetPageInfo` exactly as shown above. The field names of the wrapper type are up to you. The compiler validates that: +* the `OffsetPageInfo` type is declared and matches the definition above +* the paginated query declares both a `limit` and an `offset` argument +* the query returns multiple rows (i.e. it is not restricted to a single row) and is a query, not a subscription + +The server computes only the metadata a request actually selects, so paginated queries cost no more than unpaginated ones unless you ask for more: +* `pageSize`, `currentPage`, `hasPreviousPage`, and `prevOffset` are derived from the request arguments and cost nothing. +* `hasNextPage` and `nextOffset` make the query fetch one extra row, which is discarded before the results are returned. Without a `limit` argument the page holds every remaining row and `hasNextPage` is `false`. +* `firstEventTime` and `lastEventTime` run a second `MIN`/`MAX` query over the event time column. The compiler adds an index on that column for paginated queries. + #### Authoritative Model DataSQRL uses the GraphQL schemas the authoritative model for all API protocols. It serves as the foundational model on which operations, endpoints, and access patterns are defined. This simplifies the conceptual model and server execution since any API operation maps to a GraphQL query which is executed by a centralized and optimized GraphQL engine. diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java index 22b812a493..08ae9525cb 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlSchemaWalker.java @@ -142,8 +142,8 @@ private void walkTableFunction( var resultType = (ObjectTypeDefinition) typeDefinition; // A page wrapper ({results: [Element!], pagination: OffsetPageInfo}) is treated like a list of - // Element: validate/walk the element type against the function row type and compute pagination - // metadata via a companion count query. + // Element: validate/walk the element type against the function row type; the pagination + // metadata is computed by the server. var pagedElement = OffsetPageInfoUtil.getPagedElementType(resultType, registry); var paged = pagedElement.isPresent(); if (paged) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index 30bd38c454..c18514483c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -37,9 +37,8 @@ /** * Opt-in pagination support: a query whose result type is a page wrapper ({@code {results: - * [Element!] pagination: OffsetPageInfo}}) returns its rows plus pagination metadata computed from - * a companion COUNT(*) query. This util detects the wrapper shape and validates the user-declared - * {@code OffsetPageInfo} type. + * [Element!] pagination: OffsetPageInfo}}) returns its rows plus pagination metadata. This util + * detects the wrapper shape and validates the user-declared {@code OffsetPageInfo} type. */ public final class OffsetPageInfoUtil { diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java new file mode 100644 index 0000000000..561b70f152 --- /dev/null +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java @@ -0,0 +1,166 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.server.jdbc; + +import com.datasqrl.server.PaginationType; +import com.datasqrl.server.graphql.RootGraphQLModel.SqlQuery; +import graphql.schema.DataFetchingEnvironment; +import graphql.schema.GraphQLList; +import graphql.schema.GraphQLNonNull; +import graphql.schema.GraphQLObjectType; +import graphql.schema.GraphQLOutputType; +import io.vertx.core.json.JsonObject; +import java.util.List; + +/** + * A {@link PaginationType#OFFSET_PAGE_INFO} request: the rows of a page plus the {@code + * OffsetPageInfo} metadata around them. + * + *

Everything here is decided up front from the GraphQL request and computed in memory - it runs + * no queries. {@link VertxQueryExecutionContext} executes what {@link #pageQuery} and {@link + * #needsEventTimes} ask for and hands the rows back to {@link #toPage}: + * + *

    + *
  1. {@link #from} reads the selection set once. Which metadata fields were selected is the only + * thing that decides what has to be executed. + *
  2. {@link #pageQuery} is the same limit/offset query {@link PaginationType#LIMIT_AND_OFFSET} + * runs, fetching one extra row when {@code hasNextPage}/{@code nextOffset} were selected - + * that extra row is how they are answered without a COUNT. + *
  3. {@link #toPage} trims that extra row back off and assembles {@code {results, pagination}}. + *
+ */ +final class OffsetPageInfoQuery { + + private static final String FIRST_EVENT_TIME_COLUMN = "first_event_time"; + private static final String LAST_EVENT_TIME_COLUMN = "last_event_time"; + + private final PageRequest page; + private final PageFields fields; + private final boolean needsNextPage; + private final boolean needsEventTimes; + + private OffsetPageInfoQuery( + PageRequest page, PageFields fields, boolean needsNextPage, boolean needsEventTimes) { + this.page = page; + this.fields = fields; + this.needsNextPage = needsNextPage; + this.needsEventTimes = needsEventTimes; + } + + static OffsetPageInfoQuery from(DataFetchingEnvironment environment) { + var fields = pageFields(environment.getFieldType()); + var pagination = fields.pagination(); + var selection = environment.getSelectionSet(); + return new OffsetPageInfoQuery( + PageRequest.from(environment), + fields, + selection.containsAnyOf(pagination + "/hasNextPage", pagination + "/nextOffset"), + selection.containsAnyOf(pagination + "/firstEventTime", pagination + "/lastEventTime")); + } + + /** + * The query returning the page rows, over-fetching by one row when a next page is in question. + */ + PageRequest.BoundQuery pageQuery(SqlQuery query, List baseParams) { + return (fetchesExtraRow() ? page.plusOneRow() : page).applyTo(query, baseParams); + } + + /** Whether the caller has to run the MIN/MAX rowtime aggregate to answer this request. */ + boolean needsEventTimes() { + return needsEventTimes; + } + + /** + * Assembles the page from the rows the caller fetched. {@code eventTimes} is the single row of + * the rowtime aggregate, or an empty object when it did not run. + */ + JsonObject toPage(List rows, JsonObject eventTimes) { + Boolean hasNextPage = null; + if (needsNextPage) { + hasNextPage = fetchesExtraRow() && rows.size() > page.limit(); + if (hasNextPage) { + rows = rows.subList(0, page.limit()); + } + } + + var pagination = + paginationMetadata( + page.pageSize(rows.size()), + page.offset(), + hasNextPage, + eventTimes.getValue(FIRST_EVENT_TIME_COLUMN), + eventTimes.getValue(LAST_EVENT_TIME_COLUMN)); + + return new JsonObject().put(fields.results(), rows).put(fields.pagination(), pagination); + } + + /** + * {@code hasNextPage}/{@code nextOffset} are derived from one over-fetched row. An unbounded page + * holds every remaining row, so there is no next page and nothing to over-fetch. + */ + private boolean fetchesExtraRow() { + return needsNextPage && !page.unbounded(); + } + + /** + * Builds the {@code OffsetPageInfo} object. A null {@code hasNextPage} means the request did not + * select the next-page fields, so they are left out entirely - GraphQL never reads them. + */ + static JsonObject paginationMetadata( + int pageSize, int offset, Boolean hasNextPage, Object firstEventTime, Object lastEventTime) { + var hasPreviousPage = offset > 0; + var pagination = + new JsonObject() + .put("pageSize", pageSize) + .put("currentPage", pageSize == 0 ? 1 : offset / pageSize + 1) + .put("hasPreviousPage", hasPreviousPage) + .put( + "prevOffset", + hasPreviousPage ? Integer.valueOf(Math.max(0, offset - pageSize)) : null) + .put("firstEventTime", firstEventTime) + .put("lastEventTime", lastEventTime); + + if (hasNextPage != null) { + pagination + .put("hasNextPage", hasNextPage) + .put("nextOffset", hasNextPage ? Integer.valueOf(offset + pageSize) : null); + } + return pagination; + } + + /** The results/pagination field names of the page wrapper type this request returns. */ + private static PageFields pageFields(GraphQLOutputType fieldType) { + var objectType = (GraphQLObjectType) unwrapNonNull(fieldType); + String results = null; + String pagination = null; + for (var field : objectType.getFieldDefinitions()) { + if (unwrapNonNull(field.getType()) instanceof GraphQLList) { + results = field.getName(); + } else { + pagination = field.getName(); + } + } + return new PageFields(results, pagination); + } + + private static GraphQLOutputType unwrapNonNull(GraphQLOutputType type) { + return type instanceof GraphQLNonNull nonNull + ? (GraphQLOutputType) nonNull.getWrappedType() + : type; + } + + private record PageFields(String results, String pagination) {} +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/PageRequest.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/PageRequest.java new file mode 100644 index 0000000000..d8524d650a --- /dev/null +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/PageRequest.java @@ -0,0 +1,83 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.server.jdbc; + +import static com.datasqrl.server.jdbc.SchemaConstants.LIMIT; +import static com.datasqrl.server.jdbc.SchemaConstants.OFFSET; + +import com.datasqrl.server.PaginationType; +import com.datasqrl.server.graphql.RootGraphQLModel.SqlQuery; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * The {@code limit}/{@code offset} arguments of one request. A null limit means the caller did not + * ask for one, so the query returns every remaining row. + * + *

This is the only piece {@link PaginationType#LIMIT_AND_OFFSET} and {@link + * PaginationType#OFFSET_PAGE_INFO} share: both slice the result the same way, they differ only in + * what they return around the rows. + */ +record PageRequest(Integer limit, int offset) { + + /** Bind parameters must be a number; this stands in for "no limit" when binding. */ + private static final int NO_LIMIT = Integer.MAX_VALUE; + + static PageRequest from(DataFetchingEnvironment environment) { + Integer limit = environment.getArgument(LIMIT); + int offset = Optional.ofNullable(environment.getArgument(OFFSET)).orElse(0); + return new PageRequest(limit, offset); + } + + boolean unbounded() { + return limit == null; + } + + /** The number of rows on the page, which for an unbounded request is whatever came back. */ + int pageSize(int rowCount) { + return unbounded() ? rowCount : limit; + } + + /** Same page, one row longer. An unbounded page already holds every row, so it is unchanged. */ + PageRequest plusOneRow() { + return unbounded() ? this : new PageRequest(limit + 1, offset); + } + + /** + * Applies this page to {@code query}. Databases that support binding get limit/offset appended as + * bind parameters, the others (e.g. Snowflake) get them written into the SQL text. {@code + * baseParams} is never modified: the caller may still need it for a companion query. + */ + BoundQuery applyTo(SqlQuery query, List baseParams) { + if (!query.getDatabase().supportsLimitOffsetBinding) { + var sql = + AbstractQueryExecutionContext.addLimitOffsetToQuery( + query.getSql(), unbounded() ? "ALL" : String.valueOf(limit), String.valueOf(offset)); + return new BoundQuery(sql, baseParams); + } + + // ArrayList rather than List.copyOf: parameter values may be null + var params = new ArrayList<>(baseParams); + params.add(unbounded() ? NO_LIMIT : limit); + params.add(offset); + return new BoundQuery(query.getSql(), params); + } + + /** The SQL to execute and the full parameter list to bind to it. */ + record BoundQuery(String sql, List params) {} +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index c4e80618b7..81d4dd169d 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -15,27 +15,19 @@ */ package com.datasqrl.server.jdbc; -import static com.datasqrl.server.jdbc.SchemaConstants.LIMIT; -import static com.datasqrl.server.jdbc.SchemaConstants.OFFSET; - -import com.datasqrl.server.PaginationType; import com.datasqrl.server.VertxServerContext; import com.datasqrl.server.graphql.RootGraphQLModel; import com.datasqrl.server.graphql.RootGraphQLModel.Argument; import com.datasqrl.server.graphql.RootGraphQLModel.ResolvedSqlQuery; import com.datasqrl.server.graphql.RootGraphQLModel.SqlQuery; +import com.datasqrl.server.jdbc.PageRequest.BoundQuery; import graphql.schema.DataFetchingEnvironment; -import graphql.schema.GraphQLList; -import graphql.schema.GraphQLNonNull; -import graphql.schema.GraphQLObjectType; -import graphql.schema.GraphQLOutputType; import io.vertx.core.Future; import io.vertx.core.json.JsonObject; import io.vertx.sqlclient.Row; import io.vertx.sqlclient.RowSet; import io.vertx.sqlclient.Tuple; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.StreamSupport; @@ -45,6 +37,9 @@ * queries (paginated or not) in Vert.x and mapping the database resultSet to json for using in * GraphQL responses. It also implements the parameters and arguments visitors for the {@link * RootGraphQLModel} visitors + * + *

Each pagination type has its own execution path; the pagination logic itself lives in {@link + * PageRequest} and {@link OffsetPageInfoQuery}, this class only runs queries and maps rows. */ public class VertxQueryExecutionContext extends AbstractQueryExecutionContext { @@ -62,237 +57,98 @@ public VertxQueryExecutionContext( @Override public CompletableFuture runQuery(ResolvedSqlQuery resolvedQuery, boolean isList) { - getParamArgumentsFuture(resolvedQuery.getQuery().getParameters()) + var query = resolvedQuery.getQuery(); + getParamArgumentsFuture(query.getParameters()) .whenComplete( - (paramObj, throwable) -> { + (params, throwable) -> { if (throwable != null) { cf.completeExceptionally(throwable); - } else if (resolvedQuery.getQuery().getPagination() - == PaginationType.OFFSET_PAGE_INFO) { - runPaginatedQuery(resolvedQuery, paramObj); - } else { - runPlainQuery(resolvedQuery, isList, paramObj); + return; + } + switch (query.getPagination()) { + case NONE -> + runRowQuery(resolvedQuery, new BoundQuery(query.getSql(), params), isList); + case LIMIT_AND_OFFSET -> + runRowQuery( + resolvedQuery, + PageRequest.from(environment).applyTo(query, params), + isList); + case OFFSET_PAGE_INFO -> runOffsetPageInfoQuery(resolvedQuery, params); + default -> + cf.completeExceptionally( + new UnsupportedOperationException( + "Unsupported pagination: " + query.getPagination())); } }); return cf; } - /** Executes a bare (non-paged) query, applying limit/offset when the query declares them. */ - private void runPlainQuery( - ResolvedSqlQuery resolvedQuery, boolean isList, List paramObj) { - var query = resolvedQuery.getQuery(); - var sql = query.getSql(); - if (query.getPagination() == PaginationType.LIMIT_AND_OFFSET) { - var limit = Optional.ofNullable(environment.getArgument(LIMIT)); - var offset = Optional.ofNullable(environment.getArgument(OFFSET)); - sql = - applyLimitOffset( - query, - sql, - paramObj, - limit.orElse(Integer.MAX_VALUE), - offset.orElse(0), - limit.isPresent()); - } - - execute(resolvedQuery, sql, paramObj) - .map(r -> resultMapper(r, isList)) + /** Completes with the rows themselves: no page wrapper, no metadata. */ + private void runRowQuery(ResolvedSqlQuery resolvedQuery, BoundQuery boundQuery, boolean isList) { + execute(resolvedQuery, boundQuery) + .map(rows -> unboxList(toJson(rows), isList)) .onSuccess(cf::complete) - .onFailure( - f -> { - f.printStackTrace(); - cf.completeExceptionally(f); - }); + .onFailure(this::failQuery); } /** - * Executes an {@link PaginationType#OFFSET_PAGE_INFO} query: the page data query always runs, - * while the pagination metadata is computed lazily from the selection set. The event-time - * aggregate runs only when an event-time field is selected; {@code hasNextPage}/{@code - * nextOffset} are answered by fetching one extra row instead of any aggregate. + * Completes with a page: the page query always runs, the rowtime aggregate behind {@code + * firstEventTime}/{@code lastEventTime} only when the request selected them and the query has a + * rowtime column. {@link OffsetPageInfoQuery} decides both and assembles the response. */ - private void runPaginatedQuery(ResolvedSqlQuery resolvedQuery, List paramObj) { + private void runOffsetPageInfoQuery(ResolvedSqlQuery resolvedQuery, List params) { var query = resolvedQuery.getQuery(); - var fieldNames = pageFieldNames(environment.getFieldType()); - var pag = fieldNames.paginationField(); - var selection = environment.getSelectionSet(); + var pageInfoQuery = OffsetPageInfoQuery.from(environment); - var needEventTimes = selection.containsAnyOf(pag + "/firstEventTime", pag + "/lastEventTime"); - var eventTimesSql = needEventTimes ? query.getEventTimesSql() : null; + var pageFuture = execute(resolvedQuery, pageInfoQuery.pageQuery(query, params)); + var eventTimesFuture = + pageInfoQuery.needsEventTimes() && query.getEventTimesSql() != null + ? executeEventTimes(query, params) + : Future.>succeededFuture(null); - var limit = Optional.ofNullable(environment.getArgument(LIMIT)); - var offset = Optional.ofNullable(environment.getArgument(OFFSET)); - var limitValue = limit.orElse(Integer.MAX_VALUE); - var offsetValue = offset.orElse(0); - - // hasNextPage/nextOffset are derived by fetching one extra row. Without a limit the page holds - // every remaining row, so there is no next page and no extra row to fetch. - var deriveNext = selection.containsAnyOf(pag + "/hasNextPage", pag + "/nextOffset"); - var fetchExtraRow = deriveNext && limitValue != Integer.MAX_VALUE; - var fetchLimit = fetchExtraRow ? limitValue + 1 : limitValue; - - // The aggregate binds the base parameters only. Tuple.from snapshots the list here (before - // limit/offset are appended below) and, unlike List.copyOf, tolerates null bind values. - var aggregateParams = eventTimesSql != null ? Tuple.from(paramObj) : null; - - var sql = - applyLimitOffset( - query, query.getSql(), paramObj, fetchLimit, offsetValue, limit.isPresent()); - - var dataFuture = execute(resolvedQuery, sql, paramObj); - Future> aggregateFuture = - eventTimesSql == null - ? Future.succeededFuture(null) - : serverContext - .getSqlClient() - .execute(query.getDatabase(), eventTimesSql, aggregateParams); - - Future.all(dataFuture, aggregateFuture) + Future.all(pageFuture, eventTimesFuture) .map( - c -> - pagedResultMapper( - dataFuture.result(), - aggregateFuture.result(), - fieldNames, - limitValue, - offsetValue, - fetchExtraRow, - deriveNext)) + ignored -> + pageInfoQuery.toPage( + toJson(pageFuture.result()), firstRowAsJson(eventTimesFuture.result()))) .onSuccess(cf::complete) - .onFailure( - f -> { - f.printStackTrace(); - cf.completeExceptionally(f); - }); - } - - /** - * Binds limit/offset as parameters (databases that support it) or rewrites the SQL text - * (databases that don't, e.g. Snowflake). Returns the SQL to execute. - */ - private static String applyLimitOffset( - SqlQuery query, - String sql, - List paramObj, - int limit, - int offset, - boolean limitPresent) { - if (!query.getDatabase().supportsLimitOffsetBinding) { - return AbstractQueryExecutionContext.addLimitOffsetToQuery( - sql, limitPresent ? String.valueOf(limit) : "ALL", String.valueOf(offset)); - } - paramObj.add(limit); - paramObj.add(offset); - return sql; + .onFailure(this::failQuery); } - private Future> execute( - ResolvedSqlQuery resolvedQuery, String sql, List paramObj) { + private Future> execute(ResolvedSqlQuery resolvedQuery, BoundQuery boundQuery) { var container = (PreparedVertxSqrlQuery) resolvedQuery.getPreparedQueryContainer(); - var params = Tuple.from(paramObj); + var params = Tuple.from(boundQuery.params()); if (container == null) { return serverContext .getSqlClient() - .execute(resolvedQuery.getQuery().getDatabase(), sql, params); + .execute(resolvedQuery.getQuery().getDatabase(), boundQuery.sql(), params); } return serverContext.getSqlClient().execute(container.preparedQuery(), params); } - private Object resultMapper(RowSet r, boolean isList) { - var o = StreamSupport.stream(r.spliterator(), false).map(Row::toJson).toList(); - - return unboxList(o, isList); + /** The rowtime aggregate is never prepared and binds the base parameters only. */ + private Future> executeEventTimes(SqlQuery query, List params) { + return serverContext + .getSqlClient() + .execute(query.getDatabase(), query.getEventTimesSql(), Tuple.from(params)); } - private Object pagedResultMapper( - RowSet dataRows, - RowSet eventTimeRows, - PageFieldNames fieldNames, - int limit, - int offset, - boolean extraRowFetched, - boolean deriveNextFromResults) { - var results = StreamSupport.stream(dataRows.spliterator(), false).map(Row::toJson).toList(); - - Boolean hasNextPage = null; - if (extraRowFetched) { - hasNextPage = results.size() > limit; - if (hasNextPage) { - results = results.subList(0, limit); - } - } else if (deriveNextFromResults) { - hasNextPage = false; // no limit given: the page contains every remaining row - } - - Object firstEventTime = null; - Object lastEventTime = null; - if (eventTimeRows != null) { - var it = eventTimeRows.iterator(); - var eventTimeJson = it.hasNext() ? it.next().toJson() : new JsonObject(); - firstEventTime = eventTimeJson.getValue("first_event_time"); - lastEventTime = eventTimeJson.getValue("last_event_time"); - } - - var pagination = - buildPaginationMetadata(hasNextPage, limit, offset, firstEventTime, lastEventTime); - // An absent limit means "return everything" (limit == Integer.MAX_VALUE); report the actual - // number of rows on this page as pageSize rather than leaking the sentinel. - if (limit == Integer.MAX_VALUE) { - pagination.put("pageSize", results.size()); - } - return new JsonObject() - .put(fieldNames.resultsField(), results) - .put(fieldNames.paginationField(), pagination); + private void failQuery(Throwable throwable) { + throwable.printStackTrace(); + cf.completeExceptionally(throwable); } - /** Derives the results/pagination field names from the page wrapper's GraphQL object type. */ - private static PageFieldNames pageFieldNames(GraphQLOutputType fieldType) { - if (fieldType instanceof GraphQLNonNull g) { - fieldType = (GraphQLOutputType) g.getWrappedType(); - } - var objectType = (GraphQLObjectType) fieldType; - String resultsField = null; - String paginationField = null; - for (var field : objectType.getFieldDefinitions()) { - var type = field.getType(); - if (type instanceof GraphQLNonNull g) { - type = (GraphQLOutputType) g.getWrappedType(); - } - if (type instanceof GraphQLList) { - resultsField = field.getName(); - } else { - paginationField = field.getName(); - } - } - return new PageFieldNames(resultsField, paginationField); + private static List toJson(RowSet rows) { + return StreamSupport.stream(rows.spliterator(), false).map(Row::toJson).toList(); } - private record PageFieldNames(String resultsField, String paginationField) {} - - /** - * Builds the pagination metadata object. {@code hasNextPage} is null when the request did not - * select fields requiring it; the corresponding fields are then left out (GraphQL never reads - * unselected fields). - */ - static JsonObject buildPaginationMetadata( - Boolean hasNextPage, int limit, int offset, Object firstEventTime, Object lastEventTime) { - boolean hasPreviousPage = offset > 0; - var pagination = - new JsonObject() - .put("pageSize", limit) - .put("currentPage", limit == 0 ? 1 : offset / limit + 1) - .put("hasPreviousPage", hasPreviousPage) - .put( - "prevOffset", hasPreviousPage ? Integer.valueOf(Math.max(0, offset - limit)) : null) - .put("firstEventTime", firstEventTime) - .put("lastEventTime", lastEventTime); - - if (hasNextPage != null) { - pagination - .put("hasNextPage", hasNextPage) - .put("nextOffset", hasNextPage ? Integer.valueOf(offset + limit) : null); + /** The single row of an aggregate query, empty when the query did not run or returned nothing. */ + private static JsonObject firstRowAsJson(RowSet rows) { + if (rows == null) { + return new JsonObject(); } - return pagination; + var iterator = rows.iterator(); + return iterator.hasNext() ? iterator.next().toJson() : new JsonObject(); } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java index c51fd13696..7f7a121dad 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java @@ -23,7 +23,7 @@ class PaginationMetadataTest { @Test void givenFirstPageWithNext_whenBuildMetadata_thenHasNextNoPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(true, 10, 0, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, true, null, null); assertThat(json.getInteger("pageSize")).isEqualTo(10); assertThat(json.getInteger("currentPage")).isEqualTo(1); @@ -35,7 +35,7 @@ void givenFirstPageWithNext_whenBuildMetadata_thenHasNextNoPrevious() { @Test void givenMiddlePageWithNext_whenBuildMetadata_thenHasBothNeighbours() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(true, 10, 10, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 10, true, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(2); assertThat(json.getBoolean("hasNextPage")).isTrue(); @@ -46,7 +46,7 @@ void givenMiddlePageWithNext_whenBuildMetadata_thenHasBothNeighbours() { @Test void givenLastPage_whenBuildMetadata_thenNoNextHasPrevious() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(false, 10, 20, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 20, false, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(3); assertThat(json.getBoolean("hasNextPage")).isFalse(); @@ -57,7 +57,7 @@ void givenLastPage_whenBuildMetadata_thenNoNextHasPrevious() { @Test void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(null, 0, 0, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(0, 0, null, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(1); } @@ -65,8 +65,8 @@ void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { @Test void givenEventTimes_whenBuildMetadata_thenPassedThrough() { var json = - VertxQueryExecutionContext.buildPaginationMetadata( - null, 10, 0, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); + OffsetPageInfoQuery.paginationMetadata( + 10, 0, null, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); assertThat(json.getString("firstEventTime")).isEqualTo("2024-01-01T00:00:00Z"); assertThat(json.getString("lastEventTime")).isEqualTo("2024-01-02T00:00:00Z"); @@ -74,7 +74,7 @@ void givenEventTimes_whenBuildMetadata_thenPassedThrough() { @Test void givenNoNextPageInfoQueried_whenBuildMetadata_thenNextFieldsOmitted() { - var json = VertxQueryExecutionContext.buildPaginationMetadata(null, 10, 0, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, null, null, null); assertThat(json.containsKey("hasNextPage")).isFalse(); assertThat(json.containsKey("nextOffset")).isFalse(); From 5432f0e2dd3360e8d94b924adaa597874db47499 Mon Sep 17 00:00:00 2001 From: Ferenc Csaky Date: Fri, 24 Jul 2026 18:15:17 +0200 Subject: [PATCH 11/16] make `PagedRowTimeIndexRewriter` compatible with the general `PhysicalPlanRewriter` usage --- .../datasqrl/compile/CompilationProcess.java | 17 ++---- .../config/CompilerApiConfigImpl.java | 2 +- .../com/datasqrl/engine/PhysicalPlan.java | 4 +- .../engine/server/ServerPhysicalPlan.java | 2 +- .../plan/global/JdbcIndexOptimization.java | 11 ++-- ...er.java => PagedRowTimeIndexRewriter.java} | 52 ++++++++++++------- .../plan/global/PhysicalPlanRewriter.java | 10 +++- .../datasqrl/server/GenerateServerModel.java | 2 +- .../server/GraphqlModelGenerator.java | 6 +-- .../com/datasqrl/GraphQLValidationTest.java | 9 ++++ .../comprehensiveTest-paged-results.txt | 9 ++-- 11 files changed, 71 insertions(+), 53 deletions(-) rename sqrl-planner/src/main/java/com/datasqrl/plan/global/{PagedRowtimeIndexRewriter.java => PagedRowTimeIndexRewriter.java} (69%) diff --git a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java index b337962bdb..00fa477c2c 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java +++ b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java @@ -26,7 +26,6 @@ import com.datasqrl.error.ErrorCode; import com.datasqrl.error.ErrorCollector; import com.datasqrl.plan.MainScript; -import com.datasqrl.plan.global.PagedRowtimeIndexRewriter; import com.datasqrl.plan.global.PhysicalPlanRewriter; import com.datasqrl.plan.validate.ExecutionGoal; import com.datasqrl.planner.SqlScriptPlanner; @@ -71,8 +70,6 @@ public Pair executeCompilation(Optional testsPath) var dagBuilder = planner.getDagBuilder(); var dag = dagPlanner.optimize(dagBuilder.getDag()); var physicalPlan = dagPlanner.assemble(dag, environment); - var rewriters = ServiceLoaderDiscovery.getAll(PhysicalPlanRewriter.class); - physicalPlan = physicalPlan.applyRewriting(rewriters, environment); var mutationDatabase = physicalPlan.getMutationDatabase(); writeDeploymentArtifactsHook.run(dag, planner.getCompleteScript().toString(), mutationDatabase); @@ -106,17 +103,6 @@ public Pair executeCompilation(Optional testsPath) serverPlan.getModels().put(api.version(), model); }); - // Paginated queries run a MIN/MAX(rowtime) aggregate; index their base tables' rowtime - // column. - // Which queries are paginated is only known now (after the GraphQL walk), so this runs as a - // second rewrite pass over the already-planned database DDL. - if (!serverPlan.getPagedRowtimeTables().isEmpty()) { - physicalPlan = - physicalPlan.applyRewriting( - List.of(new PagedRowtimeIndexRewriter(serverPlan.getPagedRowtimeTables())), - environment); - } - // create test artifact if (executionGoal == ExecutionGoal.TEST) { var gqlGenerator = new GqlGenerator(serverPlan.getFunctions()); @@ -132,6 +118,9 @@ public Pair executeCompilation(Optional testsPath) } } + var rewriters = ServiceLoaderDiscovery.getAll(PhysicalPlanRewriter.class); + physicalPlan = physicalPlan.applyRewriting(rewriters, environment); + // Read database file if configured and check compatibility mainScript .getMutationDatabase() diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java b/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java index 1e3a38cb6c..76bd9545fc 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/CompilerApiConfigImpl.java @@ -59,7 +59,7 @@ public int getDefaultLimit() { @Override public boolean generatePaginatedResults() { - return sqrlConfig.asBool("paginated-results").withDefault(false).get(); + return sqrlConfig.asBool("paginated-results").get(); } public enum Endpoints { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/PhysicalPlan.java b/sqrl-planner/src/main/java/com/datasqrl/engine/PhysicalPlan.java index bf44927935..426bfc490f 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/PhysicalPlan.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/PhysicalPlan.java @@ -55,8 +55,8 @@ public PhysicalPlan applyRewriting( for (PhysicalStagePlan stagePlan : stagePlans) { var enginePlan = stagePlan.plan; for (PhysicalPlanRewriter rewriter : rewriters) { - if (rewriter.appliesTo(enginePlan)) { - enginePlan = rewriter.rewrite(enginePlan, sqrlEnv); + if (rewriter.satisfied(this) && rewriter.appliesTo(enginePlan)) { + enginePlan = rewriter.rewrite(this, enginePlan, sqrlEnv); } } builder.stagePlan(new PhysicalStagePlan(stagePlan.stage, enginePlan)); diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java b/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java index 1c8a8e83ce..b9b9883803 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/server/ServerPhysicalPlan.java @@ -52,5 +52,5 @@ public class ServerPhysicalPlan implements EnginePhysicalPlan { * Base tables of paginated queries, collected during model generation so a rowtime index can be * added to their physical tables afterwards. */ - @JsonIgnore final Set pagedRowtimeTables = new LinkedHashSet<>(); + @JsonIgnore final Set pagedRowTimeTables = new LinkedHashSet<>(); } diff --git a/sqrl-planner/src/main/java/com/datasqrl/plan/global/JdbcIndexOptimization.java b/sqrl-planner/src/main/java/com/datasqrl/plan/global/JdbcIndexOptimization.java index 78a9082c4c..dddb99949a 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/plan/global/JdbcIndexOptimization.java +++ b/sqrl-planner/src/main/java/com/datasqrl/plan/global/JdbcIndexOptimization.java @@ -16,6 +16,7 @@ package com.datasqrl.plan.global; import com.datasqrl.engine.EnginePhysicalPlan; +import com.datasqrl.engine.PhysicalPlan; import com.datasqrl.engine.database.relational.AbstractJDBCDatabaseEngine; import com.datasqrl.engine.database.relational.JdbcPhysicalPlan; import com.datasqrl.planner.Sqrl2FlinkSQLTranslator; @@ -32,14 +33,16 @@ public class JdbcIndexOptimization implements PhysicalPlanRewriter { @Override - public boolean appliesTo(EnginePhysicalPlan plan) { - return plan instanceof JdbcPhysicalPlan jpp + public boolean appliesTo(EnginePhysicalPlan enginePlan) { + return enginePlan instanceof JdbcPhysicalPlan jpp && jpp.stage().engine() instanceof AbstractJDBCDatabaseEngine; } @Override - public JdbcPhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator sqrlEnv) { - var jdbcPlan = (JdbcPhysicalPlan) plan; + public JdbcPhysicalPlan rewrite( + PhysicalPlan fullPlan, EnginePhysicalPlan enginePlan, Sqrl2FlinkSQLTranslator sqrlEnv) { + + var jdbcPlan = (JdbcPhysicalPlan) enginePlan; var engine = (AbstractJDBCDatabaseEngine) jdbcPlan.stage().engine(); /*TODO: optimize the order of primary key columns (unless primary key is explicitly defined by hint) - partition keys come first diff --git a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java similarity index 69% rename from sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java rename to sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java index 569ff1c95d..2850321e7d 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowtimeIndexRewriter.java +++ b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java @@ -16,40 +16,50 @@ package com.datasqrl.plan.global; import com.datasqrl.engine.EnginePhysicalPlan; +import com.datasqrl.engine.PhysicalPlan; import com.datasqrl.engine.database.relational.AbstractJDBCDatabaseEngine; import com.datasqrl.engine.database.relational.JdbcPhysicalPlan; import com.datasqrl.engine.database.relational.JdbcStatement; +import com.datasqrl.engine.server.ServerPhysicalPlan; import com.datasqrl.planner.Sqrl2FlinkSQLTranslator; -import com.datasqrl.planner.analyzer.TableAnalysis; +import com.google.auto.service.AutoService; import java.util.List; -import java.util.Set; +import java.util.Optional; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; /** * Adds a btree index on the rowtime column of every physical table that backs a paginated ({@code * OffsetPageInfo}) query. Those queries run a companion {@code MIN/MAX(rowtime)} aggregate for * their {@code first/lastEventTime}; the btree lets the database answer it from the index endpoints * instead of scanning the table. - * - *

Unlike {@link JdbcIndexOptimization} this rewriter is constructed with the set of paginated - * base tables (only known after the GraphQL schema walk) and is applied in a second pass. */ -@RequiredArgsConstructor -public class PagedRowtimeIndexRewriter implements PhysicalPlanRewriter { - - private final Set pagedBaseTables; +@AutoService(PhysicalPlanRewriter.class) +public class PagedRowTimeIndexRewriter implements PhysicalPlanRewriter { @Override - public boolean appliesTo(EnginePhysicalPlan plan) { - return !pagedBaseTables.isEmpty() - && plan instanceof JdbcPhysicalPlan jpp + public boolean appliesTo(EnginePhysicalPlan enginePlan) { + return enginePlan instanceof JdbcPhysicalPlan jpp && jpp.stage().engine() instanceof AbstractJDBCDatabaseEngine; } @Override - public JdbcPhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator sqrlEnv) { - var jdbcPlan = (JdbcPhysicalPlan) plan; + public boolean satisfied(PhysicalPlan fullPlan) { + return getServerPlan(fullPlan).isPresent(); + } + + @Override + public JdbcPhysicalPlan rewrite( + PhysicalPlan fullPlan, EnginePhysicalPlan enginePlan, Sqrl2FlinkSQLTranslator sqrlEnv) { + var jdbcPlan = (JdbcPhysicalPlan) enginePlan; + var serverPlan = + getServerPlan(fullPlan) + .orElseThrow(() -> new IllegalStateException("Server physical plan is missing")); + + var pagedTables = serverPlan.getPagedRowTimeTables(); + if (pagedTables.isEmpty()) { + return jdbcPlan; + } + var engine = (AbstractJDBCDatabaseEngine) jdbcPlan.stage().engine(); if (!engine.getIndexSelectorConfig().supportedIndexTypes().contains(IndexType.BTREE)) { return jdbcPlan; @@ -65,13 +75,16 @@ public JdbcPhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator for (var createTbl : jdbcPlan.tableIdMap().values()) { var engineTable = createTbl.getEngineTable(); var tableAnalysis = engineTable.tableAnalysis(); - if (!isPaged(tableAnalysis)) { + if (!pagedTables.contains(tableAnalysis) + && !pagedTables.contains(tableAnalysis.getBaseTable())) { continue; } + var rowTime = tableAnalysis.getRowTime(); if (rowTime.isEmpty()) { continue; } + var index = new IndexDefinition( engineTable.tableName(), @@ -79,15 +92,16 @@ public JdbcPhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator tableAnalysis.getRowType().getFieldNames(), -1, IndexType.BTREE); + if (existingIndexNames.add(index.getName())) { builder.statement(stmtFactory.addIndex(index)); } } + return builder.build(); } - private boolean isPaged(TableAnalysis tableAnalysis) { - return pagedBaseTables.contains(tableAnalysis) - || pagedBaseTables.contains(tableAnalysis.getBaseTable()); + private Optional getServerPlan(PhysicalPlan fullPlan) { + return fullPlan.getPlans(ServerPhysicalPlan.class).findAny(); } } diff --git a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PhysicalPlanRewriter.java b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PhysicalPlanRewriter.java index 6a92724f66..755464019c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PhysicalPlanRewriter.java +++ b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PhysicalPlanRewriter.java @@ -16,11 +16,17 @@ package com.datasqrl.plan.global; import com.datasqrl.engine.EnginePhysicalPlan; +import com.datasqrl.engine.PhysicalPlan; import com.datasqrl.planner.Sqrl2FlinkSQLTranslator; public interface PhysicalPlanRewriter { - boolean appliesTo(EnginePhysicalPlan plan); + boolean appliesTo(EnginePhysicalPlan enginePlan); - EnginePhysicalPlan rewrite(EnginePhysicalPlan plan, Sqrl2FlinkSQLTranslator sqrlEnv); + EnginePhysicalPlan rewrite( + PhysicalPlan fullPlan, EnginePhysicalPlan enginePlan, Sqrl2FlinkSQLTranslator sqrlEnv); + + default boolean satisfied(PhysicalPlan fullPlan) { + return true; + } } diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java b/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java index 764228e69f..1ab1035cf8 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GenerateServerModel.java @@ -48,7 +48,7 @@ public RootGraphQLModel generateGraphQLModel(ApiSources api, ServerPhysicalPlan new GraphqlModelGenerator( serverPlan.getFunctions(), serverPlan.getMutations(), errorCollector); graphqlModelGenerator.walkAPISource(api.schema()); - serverPlan.getPagedRowtimeTables().addAll(graphqlModelGenerator.getPagedRowtimeTables()); + serverPlan.getPagedRowTimeTables().addAll(graphqlModelGenerator.getPagedRowTimeTables()); var schema = StringSchema.builder().schema(api.schema().getDefinition()).build(); var graphSchema = converter.getSchema(schema.getSchema()); var apiConfig = configuration.getCompilerConfig().getApiConfig(); diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index dd0990f92e..749dece323 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -71,8 +71,8 @@ public class GraphqlModelGenerator extends GraphqlSchemaWalker { List mutations = new ArrayList<>(); List subscriptions = new ArrayList<>(); - /** Base tables of paginated queries, so a rowtime index can be generated for them. */ - Set pagedRowtimeTables = new LinkedHashSet<>(); + /** Base tables of paginated queries, so a row-time index can be generated for them. */ + Set pagedRowTimeTables = new LinkedHashSet<>(); private final ErrorCollector errorCollector; @@ -182,7 +182,7 @@ protected void visitQuery( var eventTimesSql = paged ? buildEventTimesSql(tableFunction, executableJdbcReadQuery.getSql()) : null; if (eventTimesSql != null) { - pagedRowtimeTables.add(tableFunction.getBaseTable()); + pagedRowTimeTables.add(tableFunction.getBaseTable()); } queryBase = new SqlQuery( diff --git a/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/GraphQLValidationTest.java b/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/GraphQLValidationTest.java index 967f344a37..c5fd47c1c1 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/GraphQLValidationTest.java +++ b/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/GraphQLValidationTest.java @@ -22,6 +22,8 @@ import com.datasqrl.util.ArgumentsProviders; import java.nio.file.Path; import java.util.function.Predicate; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -50,6 +52,13 @@ void testUseCase(Path graphQLSchema) { UseCaseTestHelper.defaultPlanDirFilter()); } + @Test + @Disabled + void specificSchema() { + var schema = USECASE_DIR.resolve("comprehensiveTest-paged-results.graphqls"); + testUseCase(schema); + } + private Predicate getBuildDirFilter() { return file -> { var fileName = file.getFileName().toString(); diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index 9869a474e4..4b2ae28d25 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -1737,8 +1737,7 @@ INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` FROM `default_catalog`.`default_database`.`UnnestOrders` ; -END; - +END >>>kafka.json { "topics" : [ @@ -1799,13 +1798,11 @@ CREATE TABLE IF NOT EXISTS "TemporalJoin" ("id" BIGINT NOT NULL, "customerid" BI CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BIGINT NOT NULL, "time" TIMESTAMP WITH TIME ZONE NOT NULL, "productid" BIGINT NOT NULL, "quantity" BIGINT NOT NULL, "discount" DOUBLE PRECISION, "newId" BIGINT NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); -CREATE INDEX IF NOT EXISTS "CustomerByTime2_btree_c4" ON "CustomerByTime2" USING btree ("timestamp"); - +CREATE INDEX IF NOT EXISTS "CustomerByTime2_btree_c4" ON "CustomerByTime2" USING btree ("timestamp") >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * FROM "ExternalOrders" AS "ExternalOrders0" - INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid"; - + INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" >>>vertx.json { "models" : { From 6888e8e2919cc0ab4f6dc579f8643844858770a0 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Wed, 29 Jul 2026 14:10:44 -0300 Subject: [PATCH 12/16] fix: Adapt paginated snapshots and JdbcStatement type import after merging main Signed-off-by: Marvin Froeder --- .../global/PagedRowTimeIndexRewriter.java | 3 +- .../clickstream-package-paginated.txt | 33 +++---- .../comprehensiveTest-paged-results.txt | 78 ++++++----------- .../comprehensiveTest-paged-userdefined.txt | 85 ++++++------------- .../clickstream-package-paginated.txt | 36 ++++---- 5 files changed, 81 insertions(+), 154 deletions(-) diff --git a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java index 2850321e7d..8894a524e3 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java +++ b/sqrl-planner/src/main/java/com/datasqrl/plan/global/PagedRowTimeIndexRewriter.java @@ -15,6 +15,7 @@ */ package com.datasqrl.plan.global; +import com.datasqrl.deployment.model.JdbcStatementModel.Type; import com.datasqrl.engine.EnginePhysicalPlan; import com.datasqrl.engine.PhysicalPlan; import com.datasqrl.engine.database.relational.AbstractJDBCDatabaseEngine; @@ -67,7 +68,7 @@ public JdbcPhysicalPlan rewrite( var stmtFactory = engine.getStatementFactory(); // Existing indexes (e.g. from JdbcIndexOptimization) may already cover the rowtime column. var existingIndexNames = - jdbcPlan.getStatementsForType(JdbcStatement.Type.INDEX).stream() + jdbcPlan.getStatementsForType(Type.INDEX).stream() .map(JdbcStatement::getName) .collect(Collectors.toSet()); diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt index 3fc2dab376..09227d4bae 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/DAGWriterJsonTest/clickstream-package-paginated.txt @@ -12,7 +12,7 @@ "name" : "base-table", "description" : "Click" } ], - "plan" : "LogicalProject(url=[$0], timestamp=[$1], userid=[$2])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "plan" : "LogicalProject(timestamp=[$0], userid=[$1], url=[$2])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", "sql" : "SELECT *\nFROM `default_catalog`.`default_database`.`Click`" }, { "id" : "access:Recommendation", @@ -65,18 +65,18 @@ "name" : "stream-root", "description" : "Click" } ], - "plan" : "LogicalWatermarkAssigner(rowtime=[timestamp], watermark=[-($1, 1000:INTERVAL SECOND)])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", - "sql" : "CREATE TEMPORARY TABLE `Click__schema` (\n `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL,\n `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL,\n `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL\n)\nWITH (\n 'connector' = 'datagen'\n);\nCREATE TABLE `Click` (\n PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED,\n WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND\n)\nWITH (\n 'connector' = 'filesystem',\n 'format' = 'flexible-json',\n 'path' = '${DATA_PATH}/click.jsonl',\n 'source.monitor-interval' = '10 sec'\n)\nLIKE `Click__schema`", + "plan" : "LogicalWatermarkAssigner(rowtime=[timestamp], watermark=[-($0, 1:INTERVAL SECOND)])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "sql" : "CREATE TEMPORARY TABLE `Click__schema` (\n `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL,\n `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL,\n `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL\n)\nWITH (\n 'connector' = 'filesystem',\n 'format' = 'flexible-json',\n 'path' = '${DATA_PATH}/click.jsonl'\n);\nCREATE TABLE `Click` (\n PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED,\n WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND\n)\nLIKE `Click__schema`", "timestamp" : "timestamp", "schema" : [ { - "name" : "url", - "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" - }, { "name" : "timestamp", "type" : "TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL" }, { "name" : "userid", "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" + }, { + "name" : "url", + "type" : "VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\" NOT NULL" } ], "primary_key" : [ "url", "userid", "timestamp" ], "row_count" : "~1e8" @@ -88,7 +88,6 @@ "connector" : { "format" : "flexible-json", "path" : "${DATA_PATH}/click.jsonl", - "source.monitor-interval" : "10 sec", "connector" : "filesystem" } }, { @@ -130,7 +129,7 @@ "name" : "sort", "description" : "[1 DESC-nulls-last, 0 ASC-nulls-first]" } ], - "plan" : "LogicalAggregate(group=[{0}], total=[COUNT()])\n LogicalProject(url=[$0])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "plan" : "LogicalAggregate(group=[{0}], total=[COUNT()])\n LogicalProject(url=[$2])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", "sql" : "CREATE VIEW `Trending` AS SELECT url, count(1) AS total\n FROM Click\n GROUP BY url ORDER BY total DESC, url ASC;\n", "timestamp" : "-", "schema" : [ { @@ -151,7 +150,7 @@ "stage" : "flink", "inputs" : [ "default_catalog.default_database.Click" ], "annotations" : [ ], - "plan" : "LogicalProject(beforeURL=[$0], afterURL=[$3], timestamp=[$4])\n LogicalJoin(condition=[AND(=($2, $5), <($1, $4), >=($1, -($4, *(10, 60000:INTERVAL MINUTE))))], joinType=[inner])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", + "plan" : "LogicalProject(beforeURL=[$2], afterURL=[$5], timestamp=[$3])\n LogicalJoin(condition=[AND(=($1, $4), <($0, $3), >=($0, -($3, *(10, 60000:INTERVAL MINUTE))))], joinType=[inner])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n LogicalTableScan(table=[[default_catalog, default_database, Click]])\n", "sql" : "CREATE VIEW `VisitAfter` AS SELECT b.url AS beforeURL, a.url AS afterURL,\n a.`timestamp` AS `timestamp`\n FROM Click b JOIN Click a ON b.userid=a.userid AND\n b.`timestamp` < a.`timestamp` AND\n b.`timestamp` >= a.`timestamp` - INTERVAL 10 MINUTE;\n", "timestamp" : "timestamp", "schema" : [ { @@ -168,22 +167,18 @@ } ] >>>pipeline_source.sqrl CREATE TEMPORARY TABLE `Click__schema` ( - `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL + `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL ) WITH ( - 'connector' = 'datagen' + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = '${DATA_PATH}/click.jsonl' ); CREATE TABLE `Click` ( PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED, - WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND -) -WITH ( - 'connector' = 'filesystem', - 'format' = 'flexible-json', - 'path' = '${DATA_PATH}/click.jsonl', - 'source.monitor-interval' = '10 sec' + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) LIKE `Click__schema`; /** Most visited pages diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index 4b2ae28d25..d3264b3364 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -1197,16 +1197,11 @@ Inputs: - default_catalog.default_database.CustomerTimeWindow >>>flink-sql-no-functions.sql -CREATE TEMPORARY TABLE `Customer__schema` ( - `customerid` BIGINT NOT NULL, - `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `lastUpdated` BIGINT NOT NULL -) -WITH ( - 'connector' = 'datagen' -); CREATE TABLE `Customer` ( + `customerid` BIGINT NOT NULL, + `email` STRING NOT NULL, + `name` STRING NOT NULL, + `lastUpdated` BIGINT NOT NULL, `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND @@ -1216,18 +1211,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `Customer__schema`; -CREATE TEMPORARY TABLE `ExternalOrders__schema` ( - `id` BIGINT NOT NULL, - `customerid` BIGINT NOT NULL, - `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `ExternalOrders` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP_LTZ(3) NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, PRIMARY KEY (`id`, `time`) NOT ENFORCED, WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) @@ -1236,18 +1225,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `ExternalOrders__schema`; -CREATE TEMPORARY TABLE `_Customer__schema` ( - `customerid` BIGINT NOT NULL, - `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `lastUpdated` BIGINT NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Customer` ( + `customerid` BIGINT NOT NULL, + `email` STRING NOT NULL, + `name` STRING NOT NULL, + `lastUpdated` BIGINT NOT NULL, `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND @@ -1257,18 +1240,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Customer__schema`; -CREATE TEMPORARY TABLE `_Orders__schema` ( - `id` BIGINT NOT NULL, - `customerid` BIGINT NOT NULL, - `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Orders` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP_LTZ(3) NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, PRIMARY KEY (`id`, `time`) NOT ENFORCED, WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) @@ -1277,19 +1254,13 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Orders__schema`; -CREATE TEMPORARY TABLE `_Product__schema` ( - `productid` BIGINT NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `description` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `category` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `_ingest_time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Product` ( + `productid` BIGINT NOT NULL, + `name` STRING NOT NULL, + `description` STRING NOT NULL, + `category` STRING NOT NULL, + `_ingest_time` TIMESTAMP_LTZ(3) NOT NULL, PRIMARY KEY (`productid`, `name`, `description`, `category`) NOT ENFORCED, WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND ) @@ -1298,8 +1269,7 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Product__schema`; +); CREATE VIEW `CustomerByTime2` AS SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` @@ -1801,8 +1771,8 @@ CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING CREATE INDEX IF NOT EXISTS "CustomerByTime2_btree_c4" ON "CustomerByTime2" USING btree ("timestamp") >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * -FROM "ExternalOrders" AS "ExternalOrders0" - INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid" +FROM "ExternalOrders" + INNER JOIN "ExplicitDistinct" ON "ExternalOrders"."customerid" = "ExplicitDistinct"."customerid" >>>vertx.json { "models" : { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 678bff3af1..e35040d472 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -1197,16 +1197,11 @@ Inputs: - default_catalog.default_database.CustomerTimeWindow >>>flink-sql-no-functions.sql -CREATE TEMPORARY TABLE `Customer__schema` ( - `customerid` BIGINT NOT NULL, - `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `lastUpdated` BIGINT NOT NULL -) -WITH ( - 'connector' = 'datagen' -); CREATE TABLE `Customer` ( + `customerid` BIGINT NOT NULL, + `email` STRING NOT NULL, + `name` STRING NOT NULL, + `lastUpdated` BIGINT NOT NULL, `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND @@ -1216,18 +1211,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `Customer__schema`; -CREATE TEMPORARY TABLE `ExternalOrders__schema` ( - `id` BIGINT NOT NULL, - `customerid` BIGINT NOT NULL, - `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `ExternalOrders` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP_LTZ(3) NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, PRIMARY KEY (`id`, `time`) NOT ENFORCED, WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) @@ -1236,18 +1225,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `ExternalOrders__schema`; -CREATE TEMPORARY TABLE `_Customer__schema` ( - `customerid` BIGINT NOT NULL, - `email` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `lastUpdated` BIGINT NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Customer` ( + `customerid` BIGINT NOT NULL, + `email` STRING NOT NULL, + `name` STRING NOT NULL, + `lastUpdated` BIGINT NOT NULL, `timestamp` AS COALESCE(`TO_TIMESTAMP_LTZ`(`lastUpdated`, 0), TIMESTAMP '1970-01-01 00:00:00.000'), PRIMARY KEY (`customerid`, `lastUpdated`) NOT ENFORCED, WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND @@ -1257,18 +1240,12 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Customer__schema`; -CREATE TEMPORARY TABLE `_Orders__schema` ( - `id` BIGINT NOT NULL, - `customerid` BIGINT NOT NULL, - `time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Orders` ( + `id` BIGINT NOT NULL, + `customerid` BIGINT NOT NULL, + `time` TIMESTAMP_LTZ(3) NOT NULL, + `entries` ROW(`productid` BIGINT NOT NULL, `quantity` BIGINT NOT NULL, `unit_price` DOUBLE NOT NULL, `discount` DOUBLE) NOT NULL ARRAY NOT NULL, PRIMARY KEY (`id`, `time`) NOT ENFORCED, WATERMARK FOR `time` AS `time` - INTERVAL '0.001' SECOND ) @@ -1277,19 +1254,13 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Orders__schema`; -CREATE TEMPORARY TABLE `_Product__schema` ( - `productid` BIGINT NOT NULL, - `name` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `description` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `category` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, - `_ingest_time` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL -) -WITH ( - 'connector' = 'datagen' ); CREATE TABLE `_Product` ( + `productid` BIGINT NOT NULL, + `name` STRING NOT NULL, + `description` STRING NOT NULL, + `category` STRING NOT NULL, + `_ingest_time` TIMESTAMP_LTZ(3) NOT NULL, PRIMARY KEY (`productid`, `name`, `description`, `category`) NOT ENFORCED, WATERMARK FOR `_ingest_time` AS `_ingest_time` - INTERVAL '0.001' SECOND ) @@ -1298,8 +1269,7 @@ WITH ( 'format' = 'flexible-json', 'path' = 'file:/mock', 'source.monitor-interval' = '10 sec' -) -LIKE `_Product__schema`; +); CREATE VIEW `CustomerByTime2` AS SELECT `customerid`, `email`, `name`, `lastUpdated`, `timestamp` @@ -1737,8 +1707,7 @@ INSERT INTO `default_catalog`.`default_database`.`UnnestOrders_16` SELECT `id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`, `hash_columns`(`id`, `customerid`, `time`, `productid`, `quantity`, `discount`, `newId`) AS `__pk_hash` FROM `default_catalog`.`default_database`.`UnnestOrders` ; -END; - +END >>>kafka.json { "topics" : [ @@ -1800,13 +1769,11 @@ CREATE TABLE IF NOT EXISTS "UnnestOrders" ("id" BIGINT NOT NULL, "customerid" BI CREATE INDEX IF NOT EXISTS "SelectCustomers_hash_c2" ON "SelectCustomers" USING hash ("name"); CREATE INDEX IF NOT EXISTS "Customer_btree_c4" ON "Customer" USING btree ("timestamp"); -CREATE INDEX IF NOT EXISTS "SelectCustomers_btree_c4" ON "SelectCustomers" USING btree ("timestamp"); - +CREATE INDEX IF NOT EXISTS "SelectCustomers_btree_c4" ON "SelectCustomers" USING btree ("timestamp") >>>postgres-views.sql CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * -FROM "ExternalOrders" AS "ExternalOrders0" - INNER JOIN "ExplicitDistinct" AS "ExplicitDistinct0" ON "ExternalOrders0"."customerid" = "ExplicitDistinct0"."customerid"; - +FROM "ExternalOrders" + INNER JOIN "ExplicitDistinct" ON "ExternalOrders"."customerid" = "ExplicitDistinct"."customerid" >>>vertx.json { "models" : { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt index 6f61374a5b..a5f2908de7 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt @@ -1,8 +1,8 @@ >>>inferred_schema.graphqls type Click { - url: String! timestamp: DateTime! userid: String! + url: String! } type ClickPage { @@ -112,9 +112,9 @@ Timestamp: timestamp Row count: ~1e8 --- Schema: - - url: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL - timestamp: TIMESTAMP_LTZ(3) *ROWTIME* NOT NULL - userid: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL + - url: VARCHAR(2147483647) CHARACTER SET "UTF-16LE" NOT NULL Inputs: - default_catalog.default_database.Click__base Annotations: @@ -170,22 +170,18 @@ Inputs: >>>flink-sql-no-functions.sql CREATE TEMPORARY TABLE `Click__schema` ( - `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, - `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL + `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL ) WITH ( - 'connector' = 'datagen' + 'connector' = 'filesystem', + 'format' = 'flexible-json', + 'path' = '${DATA_PATH}/click.jsonl' ); CREATE TABLE `Click` ( PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED, - WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND -) -WITH ( - 'connector' = 'filesystem', - 'format' = 'flexible-json', - 'path' = '${DATA_PATH}/click.jsonl', - 'source.monitor-interval' = '10 sec' + WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '0.001' SECOND ) LIKE `Click__schema`; CREATE VIEW `Trending` @@ -204,9 +200,9 @@ SELECT `beforeURL` AS `url`, `afterURL` AS `rec`, COUNT(1) AS `frequency` FROM `VisitAfter` GROUP BY `beforeURL`, `afterURL`; CREATE TABLE `Click_1` ( - `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, `timestamp` TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL, `userid` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, + `url` VARCHAR(2147483647) CHARACTER SET `UTF-16LE` NOT NULL, PRIMARY KEY (`url`, `userid`, `timestamp`) NOT ENFORCED ) WITH ( @@ -278,17 +274,15 @@ INSERT INTO `default_catalog`.`default_database`.`VisitAfter_4` SELECT `beforeURL`, `afterURL`, `timestamp`, `hash_columns`(`beforeURL`, `afterURL`, `timestamp`) AS `__pk_hash` FROM `default_catalog`.`default_database`.`VisitAfter` ; -END; - +END >>>postgres-schema.sql -CREATE TABLE IF NOT EXISTS "Click" ("url" TEXT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "userid" TEXT NOT NULL, PRIMARY KEY ("url","userid","timestamp")); +CREATE TABLE IF NOT EXISTS "Click" ("timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "userid" TEXT NOT NULL, "url" TEXT NOT NULL, PRIMARY KEY ("url","userid","timestamp")); CREATE TABLE IF NOT EXISTS "Recommendation" ("url" TEXT NOT NULL, "rec" TEXT NOT NULL, "frequency" BIGINT NOT NULL, PRIMARY KEY ("url","rec")); CREATE TABLE IF NOT EXISTS "Trending" ("url" TEXT NOT NULL, "total" BIGINT NOT NULL, PRIMARY KEY ("url")); CREATE TABLE IF NOT EXISTS "VisitAfter" ("beforeURL" TEXT NOT NULL, "afterURL" TEXT NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "__pk_hash" TEXT, PRIMARY KEY ("__pk_hash")); -CREATE INDEX IF NOT EXISTS "Click_btree_c1" ON "Click" USING btree ("timestamp"); -CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("timestamp"); - +CREATE INDEX IF NOT EXISTS "Click_btree_c0" ON "Click" USING btree ("timestamp"); +CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("timestamp") >>>vertx.json { "models" : { @@ -428,7 +422,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t }, "format" : "JSON", "apiQuery" : { - "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\nurl\ntimestamp\nuserid\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\ntimestamp\nuserid\nurl\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Click", "operationType" : "QUERY" }, @@ -525,7 +519,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t ], "schema" : { "type" : "string", - "schema" : "type Click {\n url: String!\n timestamp: DateTime!\n userid: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" + "schema" : "type Click {\n timestamp: DateTime!\n userid: String!\n url: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" } } } From 4b78150a345fbd4b5bf7bd555b67b270065981f2 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Sat, 8 Aug 2026 10:30:56 -0300 Subject: [PATCH 13/16] feat: Restore totalRecords/totalPages pagination metadata behind an opt-in COUNT query Signed-off-by: Marvin Froeder --- documentation/docs/interface.md | 7 +++ .../server/GraphqlModelGenerator.java | 12 ++++- .../datasqrl/server/OffsetPageInfoUtil.java | 4 ++ .../server/graphql/RootGraphQLModel.java | 11 ++++- .../server/jdbc/OffsetPageInfoQuery.java | 46 +++++++++++++++---- .../jdbc/VertxQueryExecutionContext.java | 28 ++++++----- .../com/datasqrl/server/PagedQueryIT.java | 32 +++++++++++-- .../java/com/datasqrl/server/WriteIT.java | 1 + .../server/jdbc/PaginationMetadataTest.java | 35 +++++++++++--- ...veTest-fail-paged-no-limit-offset.graphqls | 2 + ...siveTest-fail-paged-unknown-field.graphqls | 2 + .../comprehensiveTest-paged-results.graphqls | 2 + ...mprehensiveTest-paged-userdefined.graphqls | 2 + ...hensiveTest-fail-paged-no-limit-offset.txt | 2 +- ...omprehensiveTest-fail-paged-undeclared.txt | 2 + ...eTest-fail-paged-wrong-pagination-type.txt | 2 + ...ehensiveTest-limit-offset-combinations.txt | 8 ++++ .../comprehensiveTest-paged-results.txt | 15 ++++-- .../comprehensiveTest-paged-userdefined.txt | 16 +++++-- .../comprehensiveTest-parameters-order.txt | 8 ++++ .../comprehensiveTest.txt | 8 ++++ .../clickstream-package-paginated.txt | 24 ++++++---- 22 files changed, 220 insertions(+), 49 deletions(-) diff --git a/documentation/docs/interface.md b/documentation/docs/interface.md index b4c31ec151..785379a8b0 100644 --- a/documentation/docs/interface.md +++ b/documentation/docs/interface.md @@ -82,6 +82,8 @@ type PersonPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -113,6 +115,11 @@ The server computes only the metadata a request actually selects, so paginated q * `pageSize`, `currentPage`, `hasPreviousPage`, and `prevOffset` are derived from the request arguments and cost nothing. * `hasNextPage` and `nextOffset` make the query fetch one extra row, which is discarded before the results are returned. Without a `limit` argument the page holds every remaining row and `hasNextPage` is `false`. * `firstEventTime` and `lastEventTime` run a second `MIN`/`MAX` query over the event time column. The compiler adds an index on that column for paginated queries. +* `totalRecords` and `totalPages` run a `COUNT(*)` over the entire result set, ignoring `limit`/`offset`. + +:::warning +`totalRecords` and `totalPages` are expensive: the `COUNT(*)` behind them cannot be answered from the page the request asked for and generally requires a full table scan, which gets slower as the table grows. Select them only when the client really needs an exact total, and prefer `hasNextPage`/`nextOffset` for plain "is there more?" paging. +::: #### Authoritative Model diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index 749dece323..d39b3db706 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -191,7 +191,8 @@ protected void visitQuery( pagination, executableJdbcReadQuery.getCacheDuration().toMillis(), executableJdbcReadQuery.getDatabase(), - eventTimesSql); + eventTimesSql, + paged ? buildCountSql(executableJdbcReadQuery.getSql()) : null); var coordsBuilder = ArgumentLookupQueryCoords.builder() .parentType(parentType.getName()) @@ -203,6 +204,15 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } + /** + * Builds the companion {@code COUNT(*)} query behind {@code totalRecords}/{@code totalPages}. It + * counts the whole result set, so it cannot use limit/offset and may scan the full table - which + * is why it only runs when one of those fields is selected. + */ + private static String buildCountSql(String baseSql) { + return "SELECT COUNT(*) AS \"total_records\" FROM (" + baseSql + ") x"; + } + /** * Builds the companion aggregate query computing MIN/MAX over the designated rowtime column for * {@code firstEventTime}/{@code lastEventTime}. Returns null when the result has no rowtime. The diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index c18514483c..5a76028304 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -52,6 +52,8 @@ private OffsetPageInfoUtil() {} static { PAGINATION_FIELDS.put("pageSize", "Int!"); PAGINATION_FIELDS.put("currentPage", "Int!"); + PAGINATION_FIELDS.put("totalRecords", "Long!"); + PAGINATION_FIELDS.put("totalPages", "Int!"); PAGINATION_FIELDS.put("hasNextPage", "Boolean!"); PAGINATION_FIELDS.put("hasPreviousPage", "Boolean!"); PAGINATION_FIELDS.put("nextOffset", "Int"); @@ -67,6 +69,8 @@ private OffsetPageInfoUtil() {} Map.of( "Int!", GraphQLNonNull.nonNull(Scalars.GraphQLInt), + "Long!", + GraphQLNonNull.nonNull(CustomScalars.LONG), "Boolean!", GraphQLNonNull.nonNull(Scalars.GraphQLBoolean), "Int", diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java index ad6954d140..2fca5ec302 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java @@ -308,13 +308,22 @@ public static class SqlQuery implements QueryBase { @JsonInclude(JsonInclude.Include.NON_NULL) String eventTimesSql; + /** + * Companion {@code COUNT(*)} query for {@code totalRecords}/{@code totalPages}. Only relevant + * when {@link #pagination} is {@link PaginationType#OFFSET_PAGE_INFO}, and only executed when + * the request selects one of those fields - it scans the entire result set. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + String countSql; + @Override public R accept(QueryBaseVisitor visitor, C context) { return visitor.visitSqlQuery(this, context); } public SqlQuery updateSql(String newSql) { - return new SqlQuery(newSql, parameters, pagination, cacheDurationMs, database, eventTimesSql); + return new SqlQuery( + newSql, parameters, pagination, cacheDurationMs, database, eventTimesSql, countSql); } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java index 561b70f152..5edf8ad993 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java @@ -41,23 +41,33 @@ * that extra row is how they are answered without a COUNT. *

  • {@link #toPage} trims that extra row back off and assembles {@code {results, pagination}}. * + * + *

    The one exception is {@code totalRecords}/{@code totalPages}: they need a COUNT over the whole + * result set, which the caller runs when {@link #needsTotals} asks for it. */ final class OffsetPageInfoQuery { private static final String FIRST_EVENT_TIME_COLUMN = "first_event_time"; private static final String LAST_EVENT_TIME_COLUMN = "last_event_time"; + private static final String TOTAL_RECORDS_COLUMN = "total_records"; private final PageRequest page; private final PageFields fields; private final boolean needsNextPage; private final boolean needsEventTimes; + private final boolean needsTotals; private OffsetPageInfoQuery( - PageRequest page, PageFields fields, boolean needsNextPage, boolean needsEventTimes) { + PageRequest page, + PageFields fields, + boolean needsNextPage, + boolean needsEventTimes, + boolean needsTotals) { this.page = page; this.fields = fields; this.needsNextPage = needsNextPage; this.needsEventTimes = needsEventTimes; + this.needsTotals = needsTotals; } static OffsetPageInfoQuery from(DataFetchingEnvironment environment) { @@ -68,7 +78,8 @@ static OffsetPageInfoQuery from(DataFetchingEnvironment environment) { PageRequest.from(environment), fields, selection.containsAnyOf(pagination + "/hasNextPage", pagination + "/nextOffset"), - selection.containsAnyOf(pagination + "/firstEventTime", pagination + "/lastEventTime")); + selection.containsAnyOf(pagination + "/firstEventTime", pagination + "/lastEventTime"), + selection.containsAnyOf(pagination + "/totalRecords", pagination + "/totalPages")); } /** @@ -83,11 +94,16 @@ boolean needsEventTimes() { return needsEventTimes; } + /** Whether the caller has to run the COUNT over the whole result to answer this request. */ + boolean needsTotals() { + return needsTotals; + } + /** - * Assembles the page from the rows the caller fetched. {@code eventTimes} is the single row of - * the rowtime aggregate, or an empty object when it did not run. + * Assembles the page from the rows the caller fetched. {@code eventTimes} and {@code totals} are + * the single rows of the companion aggregates, or empty objects when they did not run. */ - JsonObject toPage(List rows, JsonObject eventTimes) { + JsonObject toPage(List rows, JsonObject eventTimes, JsonObject totals) { Boolean hasNextPage = null; if (needsNextPage) { hasNextPage = fetchesExtraRow() && rows.size() > page.limit(); @@ -102,7 +118,8 @@ JsonObject toPage(List rows, JsonObject eventTimes) { page.offset(), hasNextPage, eventTimes.getValue(FIRST_EVENT_TIME_COLUMN), - eventTimes.getValue(LAST_EVENT_TIME_COLUMN)); + eventTimes.getValue(LAST_EVENT_TIME_COLUMN), + totals.getLong(TOTAL_RECORDS_COLUMN)); return new JsonObject().put(fields.results(), rows).put(fields.pagination(), pagination); } @@ -116,11 +133,17 @@ private boolean fetchesExtraRow() { } /** - * Builds the {@code OffsetPageInfo} object. A null {@code hasNextPage} means the request did not - * select the next-page fields, so they are left out entirely - GraphQL never reads them. + * Builds the {@code OffsetPageInfo} object. A null {@code hasNextPage} or {@code totalRecords} + * means the request did not select the fields deriving from it, so they are left out entirely - + * GraphQL never reads them. */ static JsonObject paginationMetadata( - int pageSize, int offset, Boolean hasNextPage, Object firstEventTime, Object lastEventTime) { + int pageSize, + int offset, + Boolean hasNextPage, + Object firstEventTime, + Object lastEventTime, + Long totalRecords) { var hasPreviousPage = offset > 0; var pagination = new JsonObject() @@ -138,6 +161,11 @@ static JsonObject paginationMetadata( .put("hasNextPage", hasNextPage) .put("nextOffset", hasNextPage ? Integer.valueOf(offset + pageSize) : null); } + if (totalRecords != null) { + pagination + .put("totalRecords", totalRecords) + .put("totalPages", pageSize == 0 ? 0 : (int) Math.ceil((double) totalRecords / pageSize)); + } return pagination; } diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index 81d4dd169d..6580437504 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -93,9 +93,11 @@ private void runRowQuery(ResolvedSqlQuery resolvedQuery, BoundQuery boundQuery, } /** - * Completes with a page: the page query always runs, the rowtime aggregate behind {@code - * firstEventTime}/{@code lastEventTime} only when the request selected them and the query has a - * rowtime column. {@link OffsetPageInfoQuery} decides both and assembles the response. + * Completes with a page: the page query always runs, the companion aggregates only when the + * request selected fields needing them - the rowtime MIN/MAX behind {@code firstEventTime}/{@code + * lastEventTime} (and only if the query has a rowtime column), and the COUNT behind {@code + * totalRecords}/{@code totalPages}. {@link OffsetPageInfoQuery} decides all of it and assembles + * the response. */ private void runOffsetPageInfoQuery(ResolvedSqlQuery resolvedQuery, List params) { var query = resolvedQuery.getQuery(); @@ -104,14 +106,20 @@ private void runOffsetPageInfoQuery(ResolvedSqlQuery resolvedQuery, List var pageFuture = execute(resolvedQuery, pageInfoQuery.pageQuery(query, params)); var eventTimesFuture = pageInfoQuery.needsEventTimes() && query.getEventTimesSql() != null - ? executeEventTimes(query, params) + ? executeAggregate(query, query.getEventTimesSql(), params) + : Future.>succeededFuture(null); + var totalsFuture = + pageInfoQuery.needsTotals() && query.getCountSql() != null + ? executeAggregate(query, query.getCountSql(), params) : Future.>succeededFuture(null); - Future.all(pageFuture, eventTimesFuture) + Future.all(pageFuture, eventTimesFuture, totalsFuture) .map( ignored -> pageInfoQuery.toPage( - toJson(pageFuture.result()), firstRowAsJson(eventTimesFuture.result()))) + toJson(pageFuture.result()), + firstRowAsJson(eventTimesFuture.result()), + firstRowAsJson(totalsFuture.result()))) .onSuccess(cf::complete) .onFailure(this::failQuery); } @@ -127,11 +135,9 @@ private Future> execute(ResolvedSqlQuery resolvedQuery, BoundQuery b return serverContext.getSqlClient().execute(container.preparedQuery(), params); } - /** The rowtime aggregate is never prepared and binds the base parameters only. */ - private Future> executeEventTimes(SqlQuery query, List params) { - return serverContext - .getSqlClient() - .execute(query.getDatabase(), query.getEventTimesSql(), Tuple.from(params)); + /** Companion aggregates are never prepared and bind the base parameters only. */ + private Future> executeAggregate(SqlQuery query, String sql, List params) { + return serverContext.getSqlClient().execute(query.getDatabase(), sql, Tuple.from(params)); } private void failQuery(Throwable throwable) { diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java index 56a1d86e4c..39f8c9274d 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -60,8 +60,8 @@ /** * Proves that pagination metadata is computed lazily from the selection set: the MIN/MAX aggregate - * query only runs when event times are selected, and {@code hasNextPage} alone is answered by - * fetching LIMIT+1 rows instead of any aggregate. + * query only runs when event times are selected, the COUNT only when totals are selected, and + * {@code hasNextPage} alone is answered by fetching LIMIT+1 rows instead of any aggregate. */ @ExtendWith(VertxExtension.class) @Testcontainers @@ -72,6 +72,8 @@ class PagedQueryIT { "SELECT MIN(\"ts\") AS \"first_event_time\", MAX(\"ts\") AS \"last_event_time\" FROM (" + BASE_SQL + ") x"; + private static final String COUNT_SQL = + "SELECT COUNT(*) AS \"total_records\" FROM (" + BASE_SQL + ") x"; @Container private static final PostgreSQLContainer postgresContainer = @@ -198,6 +200,20 @@ void givenEventTimesSelected_whenQuery_thenMinMaxAggregateRuns() { assertThat(String.valueOf(pagination.get("lastEventTime"))).startsWith("2024-01-05"); } + @Test + void givenTotalsSelected_whenQuery_thenCountAggregateRuns() { + var customers = + execute( + "{ customers(limit: 2, offset: 0) { results { customerid }" + + " pagination { totalRecords totalPages } } }"); + + assertThat(recordingClient.executed).hasSize(2); + assertThat(sqlOf(recordingClient.executed)).contains(COUNT_SQL); + assertThat(pagination(customers)) + .containsEntry("totalRecords", 5L) + .containsEntry("totalPages", 3); + } + @Test void givenNoLimitArgument_whenQuery_thenPageSizeReportsRowCountNotSentinel() { var customers = @@ -256,6 +272,7 @@ private RootGraphQLModel getPagedModel() { .schema( """ scalar DateTime + scalar Long type Query { customers(limit: Int = 10, offset: Int = 0): CustomerPage! customersUnbounded(limit: Int, offset: Int = 0): CustomerPage! @@ -271,6 +288,8 @@ private RootGraphQLModel getPagedModel() { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -293,7 +312,8 @@ private RootGraphQLModel getPagedModel() { PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - EVENT_TIMES_SQL)) + EVENT_TIMES_SQL, + COUNT_SQL)) .build()) .build()) .query( @@ -309,7 +329,8 @@ private RootGraphQLModel getPagedModel() { PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - EVENT_TIMES_SQL)) + EVENT_TIMES_SQL, + COUNT_SQL)) .build()) .build()) .query( @@ -325,7 +346,8 @@ private RootGraphQLModel getPagedModel() { PaginationType.OFFSET_PAGE_INFO, 0, DatabaseType.POSTGRES, - null)) + null, + COUNT_SQL)) .build()) .build()) .build(); diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java index 63d24ce7b1..6ffd145af3 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java @@ -172,6 +172,7 @@ private RootGraphQLModel getCustomerModel() { PaginationType.NONE, 0, DatabaseType.POSTGRES, + null, null)) .build()) .build()) diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java index 7f7a121dad..0c67b57a95 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/jdbc/PaginationMetadataTest.java @@ -23,7 +23,7 @@ class PaginationMetadataTest { @Test void givenFirstPageWithNext_whenBuildMetadata_thenHasNextNoPrevious() { - var json = OffsetPageInfoQuery.paginationMetadata(10, 0, true, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, true, null, null, null); assertThat(json.getInteger("pageSize")).isEqualTo(10); assertThat(json.getInteger("currentPage")).isEqualTo(1); @@ -35,7 +35,7 @@ void givenFirstPageWithNext_whenBuildMetadata_thenHasNextNoPrevious() { @Test void givenMiddlePageWithNext_whenBuildMetadata_thenHasBothNeighbours() { - var json = OffsetPageInfoQuery.paginationMetadata(10, 10, true, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 10, true, null, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(2); assertThat(json.getBoolean("hasNextPage")).isTrue(); @@ -46,7 +46,7 @@ void givenMiddlePageWithNext_whenBuildMetadata_thenHasBothNeighbours() { @Test void givenLastPage_whenBuildMetadata_thenNoNextHasPrevious() { - var json = OffsetPageInfoQuery.paginationMetadata(10, 20, false, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 20, false, null, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(3); assertThat(json.getBoolean("hasNextPage")).isFalse(); @@ -57,7 +57,7 @@ void givenLastPage_whenBuildMetadata_thenNoNextHasPrevious() { @Test void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { - var json = OffsetPageInfoQuery.paginationMetadata(0, 0, null, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(0, 0, null, null, null, null); assertThat(json.getInteger("currentPage")).isEqualTo(1); } @@ -66,7 +66,7 @@ void givenZeroLimit_whenBuildMetadata_thenDoesNotDivideByZero() { void givenEventTimes_whenBuildMetadata_thenPassedThrough() { var json = OffsetPageInfoQuery.paginationMetadata( - 10, 0, null, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"); + 10, 0, null, "2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z", null); assertThat(json.getString("firstEventTime")).isEqualTo("2024-01-01T00:00:00Z"); assertThat(json.getString("lastEventTime")).isEqualTo("2024-01-02T00:00:00Z"); @@ -74,11 +74,34 @@ void givenEventTimes_whenBuildMetadata_thenPassedThrough() { @Test void givenNoNextPageInfoQueried_whenBuildMetadata_thenNextFieldsOmitted() { - var json = OffsetPageInfoQuery.paginationMetadata(10, 0, null, null, null); + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, null, null, null, null); assertThat(json.containsKey("hasNextPage")).isFalse(); assertThat(json.containsKey("nextOffset")).isFalse(); assertThat(json.getInteger("pageSize")).isEqualTo(10); assertThat(json.getBoolean("hasPreviousPage")).isFalse(); } + + @Test + void givenNoTotalsQueried_whenBuildMetadata_thenTotalFieldsOmitted() { + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, null, null, null, null); + + assertThat(json.containsKey("totalRecords")).isFalse(); + assertThat(json.containsKey("totalPages")).isFalse(); + } + + @Test + void givenTotalRecords_whenBuildMetadata_thenTotalPagesRoundedUp() { + var json = OffsetPageInfoQuery.paginationMetadata(10, 0, null, null, null, 25L); + + assertThat(json.getLong("totalRecords")).isEqualTo(25L); + assertThat(json.getInteger("totalPages")).isEqualTo(3); + } + + @Test + void givenTotalRecordsAndZeroPageSize_whenBuildMetadata_thenDoesNotDivideByZero() { + var json = OffsetPageInfoQuery.paginationMetadata(0, 0, null, null, null, 25L); + + assertThat(json.getInteger("totalPages")).isZero(); + } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls index 9af1217a3a..376ca50278 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-no-limit-offset.graphqls @@ -19,6 +19,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls index bc41941f10..bb643e5061 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-fail-paged-unknown-field.graphqls @@ -20,6 +20,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls index 70a840a44a..f9997d24d6 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-results.graphqls @@ -19,6 +19,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls index 2119ac727f..d505c2c3e4 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/graphql-validation/comprehensiveTest-paged-userdefined.graphqls @@ -6,6 +6,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt index 3e2bcc6bad..833ec77664 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-no-limit-offset.txt @@ -20,7 +20,7 @@ CustomerTimeWindow := SELECT ^ [FATAL] Paginated query [CustomerByTime2] must declare both 'limit' and 'offset' arguments -in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [31:5]: +in script:comprehensiveTest-fail-paged-no-limit-offset.graphqls [33:5]: type Query { CustomerByTime2: CustomerPage! diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt index f1e03f13fe..2d500f7e5a 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-undeclared.txt @@ -23,6 +23,8 @@ CustomerTimeWindow := SELECT type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt index a7a8c6905f..0db568ec12 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-fail-paged-wrong-pagination-type.txt @@ -23,6 +23,8 @@ CustomerTimeWindow := SELECT type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt index 2630e05b72..7943328cff 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-limit-offset-combinations.txt @@ -291,6 +291,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,6 +378,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -586,6 +590,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -607,6 +613,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index d3264b3364..4b494b50df 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -291,6 +291,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,6 +378,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -586,6 +590,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -607,6 +613,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -1800,7 +1808,8 @@ FROM "ExternalOrders" "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" } } } @@ -1826,7 +1835,7 @@ FROM "ExternalOrders" }, "format" : "JSON", "apiQuery" : { - "query" : "query CustomerByTime2($limit: Int = 10, $offset: Int = 0) {\nCustomerByTime2(limit: $limit, offset: $offset) {\nresults {\ncustomerid\nemail\nname\nlastUpdated\ntimestamp\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query CustomerByTime2($limit: Int = 10, $offset: Int = 0) {\nCustomerByTime2(limit: $limit, offset: $offset) {\nresults {\ncustomerid\nemail\nname\nlastUpdated\ntimestamp\n}\npagination {\npageSize\ncurrentPage\ntotalRecords\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "CustomerByTime2", "operationType" : "QUERY" }, @@ -1837,7 +1846,7 @@ FROM "ExternalOrders" ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n}\n\ntype CustomerPage {\n results: [Customer!]\n pagination: OffsetPageInfo\n}\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n totalRecords: Long!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n CustomerByTime2(limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index e35040d472..639eab9174 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -291,6 +291,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,6 +378,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -586,6 +590,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -607,6 +613,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -1820,7 +1828,8 @@ FROM "ExternalOrders" "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" } } }, @@ -1851,7 +1860,8 @@ FROM "ExternalOrders" "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" } } } @@ -1861,7 +1871,7 @@ FROM "ExternalOrders" "operations" : [ ], "schema" : { "type" : "string", - "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" + "schema" : "\"An RFC-3339 compliant DateTime Scalar\"\nscalar DateTime\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n totalRecords: Long!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Customer {\n customerid: Long!\n email: String!\n name: String!\n lastUpdated: Long!\n timestamp: DateTime!\n related(limit: Int = 10, offset: Int = 0): CustomerRelatedPage\n}\n\ntype CustomerRelatedPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype CustomerPage {\n items: [Customer!]\n meta: OffsetPageInfo\n}\n\ntype Query {\n TableFunctionCallsTblFct(arg1: Int!, arg2: Int!, limit: Int = 10, offset: Int = 0): CustomerPage!\n}\n" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt index 89f8ca2a7b..4b125fea6f 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-parameters-order.txt @@ -291,6 +291,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,6 +378,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -586,6 +590,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -607,6 +613,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt index b1a7917ed3..ac80f771c2 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest.txt @@ -291,6 +291,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -376,6 +378,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -586,6 +590,8 @@ type CustomerPage { type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -607,6 +613,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt index a5f2908de7..cb73953740 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt @@ -28,6 +28,8 @@ scalar Long type OffsetPageInfo { pageSize: Int! currentPage: Int! + totalRecords: Long! + totalPages: Int! hasNextPage: Boolean! hasPreviousPage: Boolean! nextOffset: Int @@ -310,7 +312,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Click\") x" } } }, @@ -345,7 +348,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t ], "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, - "database" : "POSTGRES" + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"rec\", \"frequency\"\n FROM \"Recommendation\"\n ORDER BY \"url\" NULLS FIRST, \"frequency\" DESC NULLS LAST) AS \"t\"\nWHERE \"url\" = $1) x" } } }, @@ -370,7 +374,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t "parameters" : [ ], "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, - "database" : "POSTGRES" + "database" : "POSTGRES", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT \"url\", \"total\"\n FROM \"Trending\"\n ORDER BY \"total\" DESC NULLS LAST, \"url\" NULLS FIRST) AS \"t\") x" } } }, @@ -396,7 +401,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t "pagination" : "OFFSET_PAGE_INFO", "cacheDurationMs" : 0, "database" : "POSTGRES", - "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" + "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x", + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" } } } @@ -422,7 +428,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t }, "format" : "JSON", "apiQuery" : { - "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\ntimestamp\nuserid\nurl\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Click($limit: Int = 10, $offset: Int = 0) {\nClick(limit: $limit, offset: $offset) {\nresults {\ntimestamp\nuserid\nurl\n}\npagination {\npageSize\ncurrentPage\ntotalRecords\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Click", "operationType" : "QUERY" }, @@ -455,7 +461,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t }, "format" : "JSON", "apiQuery" : { - "query" : "query Recommendation($url: String!, $limit: Int = 10, $offset: Int = 0) {\nRecommendation(url: $url, limit: $limit, offset: $offset) {\nresults {\nurl\nrec\nfrequency\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Recommendation($url: String!, $limit: Int = 10, $offset: Int = 0) {\nRecommendation(url: $url, limit: $limit, offset: $offset) {\nresults {\nurl\nrec\nfrequency\n}\npagination {\npageSize\ncurrentPage\ntotalRecords\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Recommendation", "operationType" : "QUERY" }, @@ -482,7 +488,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t }, "format" : "JSON", "apiQuery" : { - "query" : "query Trending($limit: Int = 10, $offset: Int = 0) {\nTrending(limit: $limit, offset: $offset) {\nresults {\nurl\ntotal\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query Trending($limit: Int = 10, $offset: Int = 0) {\nTrending(limit: $limit, offset: $offset) {\nresults {\nurl\ntotal\n}\npagination {\npageSize\ncurrentPage\ntotalRecords\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "Trending", "operationType" : "QUERY" }, @@ -508,7 +514,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t }, "format" : "JSON", "apiQuery" : { - "query" : "query VisitAfter($limit: Int = 10, $offset: Int = 0) {\nVisitAfter(limit: $limit, offset: $offset) {\nresults {\nbeforeURL\nafterURL\ntimestamp\n}\npagination {\npageSize\ncurrentPage\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", + "query" : "query VisitAfter($limit: Int = 10, $offset: Int = 0) {\nVisitAfter(limit: $limit, offset: $offset) {\nresults {\nbeforeURL\nafterURL\ntimestamp\n}\npagination {\npageSize\ncurrentPage\ntotalRecords\ntotalPages\nhasNextPage\nhasPreviousPage\nnextOffset\nprevOffset\nfirstEventTime\nlastEventTime\n}\n}\n\n}", "queryName" : "VisitAfter", "operationType" : "QUERY" }, @@ -519,7 +525,7 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t ], "schema" : { "type" : "string", - "schema" : "type Click {\n timestamp: DateTime!\n userid: String!\n url: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" + "schema" : "type Click {\n timestamp: DateTime!\n userid: String!\n url: String!\n}\n\ntype ClickPage {\n results: [Click!]\n pagination: OffsetPageInfo\n}\n\n\"An RFC-3339 compliant Full Date Scalar\"\nscalar Date\n\n\"A DateTime scalar that handles both full RFC3339 and shorter timestamp formats\"\nscalar DateTime\n\n\"A JSON scalar\"\nscalar JSON\n\n\"24-hour clock time value string in the format `hh:mm:ss` or `hh:mm:ss.sss`.\"\nscalar LocalTime\n\n\"A 64-bit signed integer\"\nscalar Long\n\ntype OffsetPageInfo {\n pageSize: Int!\n currentPage: Int!\n totalRecords: Long!\n totalPages: Int!\n hasNextPage: Boolean!\n hasPreviousPage: Boolean!\n nextOffset: Int\n prevOffset: Int\n firstEventTime: DateTime\n lastEventTime: DateTime\n}\n\ntype Query {\n Click(limit: Int = 10, offset: Int = 0): ClickPage!\n \"Recommend pages that are visited shortly after\"\n Recommendation(\n \"the URL to get recommendations for\"\n url: String!,\n limit: Int = 10,\n offset: Int = 0\n ): RecommendationPage!\n \"Most visited pages\"\n Trending(limit: Int = 10, offset: Int = 0): TrendingPage!\n VisitAfter(limit: Int = 10, offset: Int = 0): VisitAfterPage!\n}\n\n\"Recommend pages that are visited shortly after\"\ntype Recommendation {\n url: String!\n \"the recommended page URL\"\n rec: String!\n \"the number of visitors that co-visited that page\"\n frequency: Long!\n}\n\ntype RecommendationPage {\n results: [Recommendation!]\n pagination: OffsetPageInfo\n}\n\n\"Most visited pages\"\ntype Trending {\n \"URL of the top visited page\"\n url: String!\n \"Total number of visitors\"\n total: Long!\n}\n\ntype TrendingPage {\n results: [Trending!]\n pagination: OffsetPageInfo\n}\n\ntype VisitAfter {\n beforeURL: String!\n afterURL: String!\n timestamp: DateTime!\n}\n\ntype VisitAfterPage {\n results: [VisitAfter!]\n pagination: OffsetPageInfo\n}\n\nenum _McpMethodType {\n NONE\n TOOL\n RESOURCE\n}\n\nenum _RestMethodType {\n NONE\n GET\n POST\n}\n\ndirective @api(mcp: _McpMethodType, rest: _RestMethodType, uri: String) on QUERY | MUTATION | FIELD_DEFINITION\n" } } } From aae2e5cbfb61cf2c70095062723d655e9b739be2 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Sat, 8 Aug 2026 10:38:54 -0300 Subject: [PATCH 14/16] test: Regenerate paged GraphQL snapshots with the OpenAPI artifacts from main Signed-off-by: Marvin Froeder --- .../comprehensiveTest-paged-results.txt | 81 +++++++++++++++++++ .../comprehensiveTest-paged-userdefined.txt | 23 ++++++ 2 files changed, 104 insertions(+) diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index 4b494b50df..c10351b339 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -1781,6 +1781,87 @@ CREATE INDEX IF NOT EXISTS "CustomerByTime2_btree_c4" ON "CustomerByTime2" USING CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * FROM "ExternalOrders" INNER JOIN "ExplicitDistinct" ON "ExternalOrders"."customerid" = "ExplicitDistinct"."customerid" +>>>vertx-v1-openapi.json +{ + "openapi" : "3.0.1", + "info" : { + "title" : "DataSQRL REST API", + "description" : "Auto-generated REST API documentation for DataSQRL endpoints", + "contact" : { + "name" : "DataSQRL", + "url" : "https://datasqrl.com", + "email" : "contact@datasqrl.com" + }, + "license" : { + "name" : "Apache License 2.0", + "url" : "https://www.apache.org/licenses/LICENSE-2.0" + }, + "version" : "unknown" + }, + "servers" : [ { + "url" : "http://localhost:8888", + "description" : "DataSQRL API Server" + } ], + "paths" : { + "/v1/rest/queries/CustomerByTime2" : { + "get" : { + "summary" : "GetCustomerByTime2", + "operationId" : "GetCustomerByTime2", + "parameters" : [ { + "name" : "offset", + "in" : "query", + "required" : false, + "schema" : { + "type" : "string" + } + }, { + "name" : "limit", + "in" : "query", + "required" : false, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Successful operation", + "content" : { + "application/json" : { + "schema" : { + "type" : "object", + "properties" : { + "data" : { + "type" : "object", + "description" : "Response data" + } + } + } + } + } + }, + "400" : { + "description" : "Error response", + "content" : { + "application/json" : { + "schema" : { + "type" : "object", + "properties" : { + "errors" : { + "type" : "array", + "items" : { + "type" : "object" + } + } + } + } + } + } + } + } + } + } + } +} >>>vertx.json { "models" : { diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 639eab9174..6471a353c5 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -1782,6 +1782,29 @@ CREATE INDEX IF NOT EXISTS "SelectCustomers_btree_c4" ON "SelectCustomers" USING CREATE OR REPLACE VIEW "MissedTemporalJoin"("id", "customerid", "time", "entries", "customerid0", "timestamp", "name") AS SELECT * FROM "ExternalOrders" INNER JOIN "ExplicitDistinct" ON "ExternalOrders"."customerid" = "ExplicitDistinct"."customerid" +>>>vertx-v1-openapi.json +{ + "openapi" : "3.0.1", + "info" : { + "title" : "DataSQRL REST API", + "description" : "Auto-generated REST API documentation for DataSQRL endpoints", + "contact" : { + "name" : "DataSQRL", + "url" : "https://datasqrl.com", + "email" : "contact@datasqrl.com" + }, + "license" : { + "name" : "Apache License 2.0", + "url" : "https://www.apache.org/licenses/LICENSE-2.0" + }, + "version" : "unknown" + }, + "servers" : [ { + "url" : "http://localhost:8888", + "description" : "DataSQRL API Server" + } ], + "paths" : { } +} >>>vertx.json { "models" : { From b44f94309a9a9855bf1f38d708a38a1ede0aca05 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 10 Aug 2026 09:55:28 -0300 Subject: [PATCH 15/16] perf: Answer paginated totals and event times with a single combined aggregate query Signed-off-by: Marvin Froeder --- documentation/docs/interface.md | 2 + .../server/GraphqlModelGenerator.java | 49 ++++++++++--------- .../datasqrl/server/OffsetPageInfoUtil.java | 28 +++++------ .../server/graphql/RootGraphQLModel.java | 17 ++++++- .../server/jdbc/OffsetPageInfoQuery.java | 39 ++++++++------- .../jdbc/VertxQueryExecutionContext.java | 27 ++++------ .../com/datasqrl/server/PagedQueryIT.java | 35 +++++++++++-- .../java/com/datasqrl/server/WriteIT.java | 1 + .../comprehensiveTest-paged-results.txt | 3 +- .../comprehensiveTest-paged-userdefined.txt | 6 ++- .../clickstream-package-paginated.txt | 6 ++- 11 files changed, 130 insertions(+), 83 deletions(-) diff --git a/documentation/docs/interface.md b/documentation/docs/interface.md index 785379a8b0..ebdb9a9d81 100644 --- a/documentation/docs/interface.md +++ b/documentation/docs/interface.md @@ -117,6 +117,8 @@ The server computes only the metadata a request actually selects, so paginated q * `firstEventTime` and `lastEventTime` run a second `MIN`/`MAX` query over the event time column. The compiler adds an index on that column for paginated queries. * `totalRecords` and `totalPages` run a `COUNT(*)` over the entire result set, ignoring `limit`/`offset`. +Selecting event times and totals together costs a single combined aggregate query, not two, so a request never makes more than one extra round trip. + :::warning `totalRecords` and `totalPages` are expensive: the `COUNT(*)` behind them cannot be answered from the page the request asked for and generally requires a full table scan, which gets slower as the table grows. Select them only when the client really needs an exact total, and prefer `hasNextPage`/`nextOffset` for plain "is there more?" paging. ::: diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index d39b3db706..ad98d6f04e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -179,20 +179,23 @@ protected void visitQuery( paged ? PaginationType.OFFSET_PAGE_INFO : hasLimitOrOffset ? PaginationType.LIMIT_AND_OFFSET : PaginationType.NONE; - var eventTimesSql = - paged ? buildEventTimesSql(tableFunction, executableJdbcReadQuery.getSql()) : null; - if (eventTimesSql != null) { + var baseSql = executableJdbcReadQuery.getSql(); + var eventTimes = paged ? eventTimeAggregates(tableFunction) : Optional.empty(); + if (eventTimes.isPresent()) { pagedRowTimeTables.add(tableFunction.getBaseTable()); } queryBase = new SqlQuery( - executableJdbcReadQuery.getSql(), + baseSql, parameters, pagination, executableJdbcReadQuery.getCacheDuration().toMillis(), executableJdbcReadQuery.getDatabase(), - eventTimesSql, - paged ? buildCountSql(executableJdbcReadQuery.getSql()) : null); + eventTimes.map(aggregates -> aggregateSql(aggregates, baseSql)).orElse(null), + paged ? aggregateSql(COUNT_AGGREGATE, baseSql) : null, + eventTimes + .map(aggregates -> aggregateSql(COUNT_AGGREGATE + ", " + aggregates, baseSql)) + .orElse(null)); var coordsBuilder = ArgumentLookupQueryCoords.builder() .parentType(parentType.getName()) @@ -204,35 +207,35 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } - /** - * Builds the companion {@code COUNT(*)} query behind {@code totalRecords}/{@code totalPages}. It - * counts the whole result set, so it cannot use limit/offset and may scan the full table - which - * is why it only runs when one of those fields is selected. - */ - private static String buildCountSql(String baseSql) { - return "SELECT COUNT(*) AS \"total_records\" FROM (" + baseSql + ") x"; - } + /** Select expression behind {@code totalRecords}/{@code totalPages}. */ + private static final String COUNT_AGGREGATE = "COUNT(*) AS \"total_records\""; /** - * Builds the companion aggregate query computing MIN/MAX over the designated rowtime column for - * {@code firstEventTime}/{@code lastEventTime}. Returns null when the result has no rowtime. The - * rowtime column name is the same identifier as in the base query's output. + * Select expressions computing MIN/MAX over the designated rowtime column for {@code + * firstEventTime}/{@code lastEventTime}. Empty when the result has no rowtime. The rowtime column + * name is the same identifier as in the base query's output. */ - private static String buildEventTimesSql(SqrlTableFunction tableFunction, String baseSql) { + private static Optional eventTimeAggregates(SqrlTableFunction tableFunction) { return tableFunction .getRowTime() .map(tableFunction::getField) .map(RelDataTypeField::getName) .map( col -> - "SELECT MIN(\"" + "MIN(\"" + col + "\") AS \"first_event_time\", MAX(\"" + col - + "\") AS \"last_event_time\" FROM (" - + baseSql - + ") x") - .orElse(null); + + "\") AS \"last_event_time\""); + } + + /** + * Wraps aggregate expressions around the paginated query. These aggregates cover the whole result + * set, so they cannot use limit/offset and may scan the full table - which is why the server runs + * one only when a field needing it is selected. + */ + private static String aggregateSql(String aggregates, String baseSql) { + return "SELECT " + aggregates + " FROM (" + baseSql + ") x"; } private static QueryParameterHandler convert(FunctionParameter fnParam) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java index 5a76028304..a61aa8849e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/OffsetPageInfoUtil.java @@ -18,6 +18,7 @@ import static com.datasqrl.server.util.GraphqlCheckUtil.checkState; import com.datasqrl.server.graphql.CustomScalars; +import com.google.common.collect.ImmutableMap; import graphql.Scalars; import graphql.language.FieldDefinition; import graphql.language.ListType; @@ -47,20 +48,19 @@ private OffsetPageInfoUtil() {} public static final String PAGINATION_TYPE_NAME = "OffsetPageInfo"; /** Canonical field -> printed type, kept in declaration order for the injected SDL. */ - private static final Map PAGINATION_FIELDS = new LinkedHashMap<>(); - - static { - PAGINATION_FIELDS.put("pageSize", "Int!"); - PAGINATION_FIELDS.put("currentPage", "Int!"); - PAGINATION_FIELDS.put("totalRecords", "Long!"); - PAGINATION_FIELDS.put("totalPages", "Int!"); - PAGINATION_FIELDS.put("hasNextPage", "Boolean!"); - PAGINATION_FIELDS.put("hasPreviousPage", "Boolean!"); - PAGINATION_FIELDS.put("nextOffset", "Int"); - PAGINATION_FIELDS.put("prevOffset", "Int"); - PAGINATION_FIELDS.put("firstEventTime", "DateTime"); - PAGINATION_FIELDS.put("lastEventTime", "DateTime"); - } + private static final Map PAGINATION_FIELDS = + ImmutableMap.builder() + .put("pageSize", "Int!") + .put("currentPage", "Int!") + .put("totalRecords", "Long!") + .put("totalPages", "Int!") + .put("hasNextPage", "Boolean!") + .put("hasPreviousPage", "Boolean!") + .put("nextOffset", "Int") + .put("prevOffset", "Int") + .put("firstEventTime", "DateTime") + .put("lastEventTime", "DateTime") + .build(); private static final String CANONICAL_SDL = buildCanonicalSdl(); diff --git a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java index 2fca5ec302..922813a73a 100644 --- a/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/server/graphql/RootGraphQLModel.java @@ -316,6 +316,14 @@ public static class SqlQuery implements QueryBase { @JsonInclude(JsonInclude.Include.NON_NULL) String countSql; + /** + * Variant of {@link #countSql} that also computes the {@link #eventTimesSql} aggregates, so a + * request selecting totals and event times together costs one query rather than two. Null + * whenever {@link #eventTimesSql} is. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + String countWithEventTimesSql; + @Override public R accept(QueryBaseVisitor visitor, C context) { return visitor.visitSqlQuery(this, context); @@ -323,7 +331,14 @@ public R accept(QueryBaseVisitor visitor, C context) { public SqlQuery updateSql(String newSql) { return new SqlQuery( - newSql, parameters, pagination, cacheDurationMs, database, eventTimesSql, countSql); + newSql, + parameters, + pagination, + cacheDurationMs, + database, + eventTimesSql, + countSql, + countWithEventTimesSql); } } diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java index 5edf8ad993..a14ba7342a 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java @@ -31,7 +31,7 @@ * *

    Everything here is decided up front from the GraphQL request and computed in memory - it runs * no queries. {@link VertxQueryExecutionContext} executes what {@link #pageQuery} and {@link - * #needsEventTimes} ask for and hands the rows back to {@link #toPage}: + * #aggregateSql} ask for and hands the rows back to {@link #toPage}: * *

      *
    1. {@link #from} reads the selection set once. Which metadata fields were selected is the only @@ -39,11 +39,10 @@ *
    2. {@link #pageQuery} is the same limit/offset query {@link PaginationType#LIMIT_AND_OFFSET} * runs, fetching one extra row when {@code hasNextPage}/{@code nextOffset} were selected - * that extra row is how they are answered without a COUNT. + *
    3. {@link #aggregateSql} picks the one companion query, if any, that covers the remaining + * selected fields - never more than a single extra round trip. *
    4. {@link #toPage} trims that extra row back off and assembles {@code {results, pagination}}. *
    - * - *

    The one exception is {@code totalRecords}/{@code totalPages}: they need a COUNT over the whole - * result set, which the caller runs when {@link #needsTotals} asks for it. */ final class OffsetPageInfoQuery { @@ -89,21 +88,25 @@ PageRequest.BoundQuery pageQuery(SqlQuery query, List baseParams) { return (fetchesExtraRow() ? page.plusOneRow() : page).applyTo(query, baseParams); } - /** Whether the caller has to run the MIN/MAX rowtime aggregate to answer this request. */ - boolean needsEventTimes() { - return needsEventTimes; - } - - /** Whether the caller has to run the COUNT over the whole result to answer this request. */ - boolean needsTotals() { - return needsTotals; + /** + * The single companion aggregate to run for this request, or null when the selected fields need + * none. Selecting event times and totals together picks the query computing both, so the request + * never costs more than one extra round trip - and never computes an aggregate nobody asked for. + * Event times are only available when the query has a rowtime column. + */ + String aggregateSql(SqlQuery query) { + var eventTimes = needsEventTimes && query.getEventTimesSql() != null; + if (needsTotals) { + return eventTimes ? query.getCountWithEventTimesSql() : query.getCountSql(); + } + return eventTimes ? query.getEventTimesSql() : null; } /** - * Assembles the page from the rows the caller fetched. {@code eventTimes} and {@code totals} are - * the single rows of the companion aggregates, or empty objects when they did not run. + * Assembles the page from the rows the caller fetched. {@code aggregate} is the single row of the + * companion aggregate, or an empty object when none ran. */ - JsonObject toPage(List rows, JsonObject eventTimes, JsonObject totals) { + JsonObject toPage(List rows, JsonObject aggregate) { Boolean hasNextPage = null; if (needsNextPage) { hasNextPage = fetchesExtraRow() && rows.size() > page.limit(); @@ -117,9 +120,9 @@ JsonObject toPage(List rows, JsonObject eventTimes, JsonObject total page.pageSize(rows.size()), page.offset(), hasNextPage, - eventTimes.getValue(FIRST_EVENT_TIME_COLUMN), - eventTimes.getValue(LAST_EVENT_TIME_COLUMN), - totals.getLong(TOTAL_RECORDS_COLUMN)); + aggregate.getValue(FIRST_EVENT_TIME_COLUMN), + aggregate.getValue(LAST_EVENT_TIME_COLUMN), + aggregate.getLong(TOTAL_RECORDS_COLUMN)); return new JsonObject().put(fields.results(), rows).put(fields.pagination(), pagination); } diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java index 6580437504..01d1d96c94 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/VertxQueryExecutionContext.java @@ -93,33 +93,26 @@ private void runRowQuery(ResolvedSqlQuery resolvedQuery, BoundQuery boundQuery, } /** - * Completes with a page: the page query always runs, the companion aggregates only when the - * request selected fields needing them - the rowtime MIN/MAX behind {@code firstEventTime}/{@code - * lastEventTime} (and only if the query has a rowtime column), and the COUNT behind {@code - * totalRecords}/{@code totalPages}. {@link OffsetPageInfoQuery} decides all of it and assembles - * the response. + * Completes with a page: the page query always runs, plus at most one companion aggregate - the + * rowtime MIN/MAX, the COUNT, or the query computing both - depending on which metadata fields + * the request selected. {@link OffsetPageInfoQuery} decides that and assembles the response. */ private void runOffsetPageInfoQuery(ResolvedSqlQuery resolvedQuery, List params) { var query = resolvedQuery.getQuery(); var pageInfoQuery = OffsetPageInfoQuery.from(environment); + var aggregateSql = pageInfoQuery.aggregateSql(query); var pageFuture = execute(resolvedQuery, pageInfoQuery.pageQuery(query, params)); - var eventTimesFuture = - pageInfoQuery.needsEventTimes() && query.getEventTimesSql() != null - ? executeAggregate(query, query.getEventTimesSql(), params) - : Future.>succeededFuture(null); - var totalsFuture = - pageInfoQuery.needsTotals() && query.getCountSql() != null - ? executeAggregate(query, query.getCountSql(), params) - : Future.>succeededFuture(null); + var aggregateFuture = + aggregateSql == null + ? Future.>succeededFuture(null) + : executeAggregate(query, aggregateSql, params); - Future.all(pageFuture, eventTimesFuture, totalsFuture) + Future.all(pageFuture, aggregateFuture) .map( ignored -> pageInfoQuery.toPage( - toJson(pageFuture.result()), - firstRowAsJson(eventTimesFuture.result()), - firstRowAsJson(totalsFuture.result()))) + toJson(pageFuture.result()), firstRowAsJson(aggregateFuture.result()))) .onSuccess(cf::complete) .onFailure(this::failQuery); } diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java index 39f8c9274d..a83ce5703f 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/PagedQueryIT.java @@ -60,8 +60,9 @@ /** * Proves that pagination metadata is computed lazily from the selection set: the MIN/MAX aggregate - * query only runs when event times are selected, the COUNT only when totals are selected, and - * {@code hasNextPage} alone is answered by fetching LIMIT+1 rows instead of any aggregate. + * query only runs when event times are selected, the COUNT only when totals are selected, both + * together cost a single combined aggregate, and {@code hasNextPage} alone is answered by fetching + * LIMIT+1 rows instead of any aggregate. */ @ExtendWith(VertxExtension.class) @Testcontainers @@ -74,6 +75,11 @@ class PagedQueryIT { + ") x"; private static final String COUNT_SQL = "SELECT COUNT(*) AS \"total_records\" FROM (" + BASE_SQL + ") x"; + private static final String COUNT_WITH_EVENT_TIMES_SQL = + "SELECT COUNT(*) AS \"total_records\", MIN(\"ts\") AS \"first_event_time\"," + + " MAX(\"ts\") AS \"last_event_time\" FROM (" + + BASE_SQL + + ") x"; @Container private static final PostgreSQLContainer postgresContainer = @@ -214,6 +220,22 @@ void givenTotalsSelected_whenQuery_thenCountAggregateRuns() { .containsEntry("totalPages", 3); } + @Test + void givenTotalsAndEventTimesSelected_whenQuery_thenSingleCombinedAggregateRuns() { + var customers = + execute( + "{ customers(limit: 2, offset: 0) { results { customerid }" + + " pagination { totalRecords totalPages firstEventTime lastEventTime } } }"); + + // both are answered by one aggregate rather than a COUNT and a MIN/MAX round trip + assertThat(recordingClient.executed).hasSize(2); + assertThat(sqlOf(recordingClient.executed)).contains(COUNT_WITH_EVENT_TIMES_SQL); + var pagination = pagination(customers); + assertThat(pagination).containsEntry("totalRecords", 5L).containsEntry("totalPages", 3); + assertThat(String.valueOf(pagination.get("firstEventTime"))).startsWith("2024-01-01"); + assertThat(String.valueOf(pagination.get("lastEventTime"))).startsWith("2024-01-05"); + } + @Test void givenNoLimitArgument_whenQuery_thenPageSizeReportsRowCountNotSentinel() { var customers = @@ -313,7 +335,8 @@ private RootGraphQLModel getPagedModel() { 0, DatabaseType.POSTGRES, EVENT_TIMES_SQL, - COUNT_SQL)) + COUNT_SQL, + COUNT_WITH_EVENT_TIMES_SQL)) .build()) .build()) .query( @@ -330,7 +353,8 @@ private RootGraphQLModel getPagedModel() { 0, DatabaseType.POSTGRES, EVENT_TIMES_SQL, - COUNT_SQL)) + COUNT_SQL, + COUNT_WITH_EVENT_TIMES_SQL)) .build()) .build()) .query( @@ -347,7 +371,8 @@ private RootGraphQLModel getPagedModel() { 0, DatabaseType.POSTGRES, null, - COUNT_SQL)) + COUNT_SQL, + null)) .build()) .build()) .build(); diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java index 6ffd145af3..f11a81ebce 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java +++ b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/server/WriteIT.java @@ -173,6 +173,7 @@ private RootGraphQLModel getCustomerModel() { 0, DatabaseType.POSTGRES, null, + null, null)) .build()) .build()) diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt index c10351b339..93b5b28bef 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-results.txt @@ -1890,7 +1890,8 @@ FROM "ExternalOrders" "cacheDurationMs" : 0, "database" : "POSTGRES", "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"CustomerByTime2\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"CustomerByTime2\") x" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt index 6471a353c5..b85c294ad8 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/GraphQLValidationTest/comprehensiveTest-paged-userdefined.txt @@ -1852,7 +1852,8 @@ FROM "ExternalOrders" "cacheDurationMs" : 0, "database" : "POSTGRES", "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM (SELECT *\n FROM \"Customer\"\n WHERE \"customerid\" > $2) AS \"t0\"\nWHERE \"customerid\" > $1) x" } } }, @@ -1884,7 +1885,8 @@ FROM "ExternalOrders" "cacheDurationMs" : 0, "database" : "POSTGRES", "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Customer\"\nWHERE $1 = \"customerid\"\nORDER BY \"timestamp\" NULLS FIRST) x" } } } diff --git a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt index cb73953740..ec7fec3485 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt +++ b/sqrl-testing/sqrl-testing-integration/src/test/resources/snapshots/com/datasqrl/UseCaseCompileTest/clickstream-package-paginated.txt @@ -313,7 +313,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t "cacheDurationMs" : 0, "database" : "POSTGRES", "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Click\") x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT *\nFROM \"Click\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT *\nFROM \"Click\") x" } } }, @@ -402,7 +403,8 @@ CREATE INDEX IF NOT EXISTS "VisitAfter_btree_c2" ON "VisitAfter" USING btree ("t "cacheDurationMs" : 0, "database" : "POSTGRES", "eventTimesSql" : "SELECT MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x", - "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" + "countSql" : "SELECT COUNT(*) AS \"total_records\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x", + "countWithEventTimesSql" : "SELECT COUNT(*) AS \"total_records\", MIN(\"timestamp\") AS \"first_event_time\", MAX(\"timestamp\") AS \"last_event_time\" FROM (SELECT \"beforeURL\", \"afterURL\", \"timestamp\"\nFROM \"VisitAfter\") x" } } } From ebbea3ec769c5087dab5081a58a60470805a5358 Mon Sep 17 00:00:00 2001 From: Marvin Froeder Date: Mon, 10 Aug 2026 12:10:12 -0300 Subject: [PATCH 16/16] refactor: Address review comments on pagination metadata Signed-off-by: Marvin Froeder --- .../datasqrl/server/GraphqlModelGenerator.java | 6 +++--- .../server/jdbc/OffsetPageInfoQuery.java | 16 +++------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java index ad98d6f04e..568eab0b0d 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/server/GraphqlModelGenerator.java @@ -67,6 +67,9 @@ @Getter public class GraphqlModelGenerator extends GraphqlSchemaWalker { + /** Select expression behind {@code totalRecords}/{@code totalPages}. */ + private static final String COUNT_AGGREGATE = "COUNT(*) AS \"total_records\""; + List queryCoords = new ArrayList<>(); List mutations = new ArrayList<>(); List subscriptions = new ArrayList<>(); @@ -207,9 +210,6 @@ protected void visitQuery( queryCoords.add(coordsBuilder.build()); } - /** Select expression behind {@code totalRecords}/{@code totalPages}. */ - private static final String COUNT_AGGREGATE = "COUNT(*) AS \"total_records\""; - /** * Select expressions computing MIN/MAX over the designated rowtime column for {@code * firstEventTime}/{@code lastEventTime}. Empty when the result has no rowtime. The rowtime column diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java index a14ba7342a..2bc643f2a8 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java +++ b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/server/jdbc/OffsetPageInfoQuery.java @@ -24,6 +24,8 @@ import graphql.schema.GraphQLOutputType; import io.vertx.core.json.JsonObject; import java.util.List; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; /** * A {@link PaginationType#OFFSET_PAGE_INFO} request: the rows of a page plus the {@code @@ -44,6 +46,7 @@ *
  • {@link #toPage} trims that extra row back off and assembles {@code {results, pagination}}. * */ +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) final class OffsetPageInfoQuery { private static final String FIRST_EVENT_TIME_COLUMN = "first_event_time"; @@ -56,19 +59,6 @@ final class OffsetPageInfoQuery { private final boolean needsEventTimes; private final boolean needsTotals; - private OffsetPageInfoQuery( - PageRequest page, - PageFields fields, - boolean needsNextPage, - boolean needsEventTimes, - boolean needsTotals) { - this.page = page; - this.fields = fields; - this.needsNextPage = needsNextPage; - this.needsEventTimes = needsEventTimes; - this.needsTotals = needsTotals; - } - static OffsetPageInfoQuery from(DataFetchingEnvironment environment) { var fields = pageFields(environment.getFieldType()); var pagination = fields.pagination();