Materialize/test/sqllogictest/mzcompose.py

# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.

"""
Run SQL tests using an instance of Mz that is embedded in the sqllogic binary
itself. Good for basic SQL tests, but can't interact with sources like
MySQL/Kafka, see Testdrive for that.
"""

from __future__ import annotations

import argparse
import os
import threading
from argparse import Namespace
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from queue import Queue

from materialize import buildkite, ci_util, file_util
from materialize.mzcompose.composition import Composition, WorkflowArgumentParser
from materialize.mzcompose.services.mz import Mz
from materialize.mzcompose.services.postgres import CockroachOrPostgresMetadata
from materialize.mzcompose.services.sql_logic_test import SqlLogicTest
from materialize.ui import CommandFailureCausedUIError

NUM_SLTS = 8
SLTS = [f"slt_{i+1}" for i in range(NUM_SLTS)]

SERVICES = [CockroachOrPostgresMetadata(), Mz(app_password="")] + [
    SqlLogicTest(name=slt) for slt in SLTS
]

COCKROACH_DEFAULT_PORT = 26257


def workflow_default(c: Composition) -> None:
    def process(name: str) -> None:
        if name == "default":
            return

        with c.test_case(name):
            c.workflow(name)

    c.test_parts(list(c.workflows.keys()), process)


def workflow_fast_tests(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Run fast SQL logic tests"""
    run_sqllogictest(c, parser, compileFastSltConfig())


def workflow_slow_tests(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Run slow SQL logic tests"""
    run_sqllogictest(
        c,
        parser,
        compileSlowSltConfig(),
    )


def workflow_selection(c: Composition, parser: WorkflowArgumentParser) -> None:
    """Run specific SQL logic tests using pattern"""
    parser.add_argument(
        "--pattern",
        type=str,
        action="append",
        help="Example: test/sqllogictest/map.slt",
    )

    parser.add_argument(
        "--auto-index-selects",
        default=False,
        type=bool,
        action=argparse.BooleanOptionalAction,
    )

    run_sqllogictest(
        c,
        parser,
        InputBasedSltRunConfig(),
    )


def run_sqllogictest(
    c: Composition, parser: WorkflowArgumentParser, run_config: SltRunConfig
) -> None:
    parser.add_argument("--replica-size", default=2, type=int)
    parser.add_argument("--replicas", default=1, type=int)
    args = parser.parse_args()
    c.up(c.metadata_store())

    work_queue = Queue()
    stop_event = threading.Event()

    for step in run_config.steps:
        step.configure(args)
        sharded_files: list[str] = sorted(
            buildkite.shard_list(step.file_list, lambda file: file)
        )
        for file in sharded_files:
            work_queue.put((step, file))

    # Hacky way to make sure we have downloaded the image
    c.up({"name": "slt_1", "persistent": True})

    def worker(container_name: str):
        exception: Exception | None = None
        while True:
            if stop_event.is_set():
                return

            try:
                step, file = work_queue.get_nowait()
            except Exception:
                break  # Queue is empty

            if "singlereplica_" in file and args.replicas > 1:
                continue

            junit_report_path = ci_util.junit_report_filename(
                f"{c.name}-{file.replace('.', '_').replace('/', '_')}"
            )
            cmd = step.to_command(
                container_name,
                file,
                args.replicas,
                args.replica_size,
                junit_report_path,
                c.metadata_store(),
            )
            try:
                c.run(container_name, *cmd, capture=True, capture_stderr=True)
                # Uploading successful junit files wastes time and contains no useful information
                os.remove(junit_report_path)
            except CommandFailureCausedUIError as e:
                print(f"+++ {file} failed, STDOUT:\n{e.stdout}\nSTDERR:\n{e.stderr}")
                exception = e
            finally:
                work_queue.task_done()
        if exception:
            raise exception

    with ThreadPoolExecutor(max_workers=len(SLTS)) as executor:
        futures = [executor.submit(worker, container_name) for container_name in SLTS]
        try:
            for future in as_completed(futures):
                future.result()
        except Exception:
            stop_event.set()
            raise


class SltRunConfig:
    def __init__(self):
        self.steps: list[SltRunStepConfig] = []


class SltRunStepConfig:
    def __init__(self, file_set: set[str], flags: list[str]):
        file_list = list(file_set)
        file_list.sort()
        self.file_list = file_list
        self.flags = flags

    def configure(self, args: Namespace) -> None:
        pass

    def to_command(
        self,
        container_name: str,
        file: str,
        replicas: int,
        replica_size: int,
        junit_report_path: Path,
        metadata_store: str,
        metadata_store_port: int = COCKROACH_DEFAULT_PORT,
    ) -> list[str]:
        sqllogictest_config = [
            f"--junit-report={junit_report_path}",
            f"--postgres-url=postgres://root@{metadata_store}:{metadata_store_port}",
            f"--prefix={container_name}",
            f"--replica-size={replica_size}",
            f"--replicas={replicas}",
        ]
        command = [
            "sqllogictest",
            "-v",
            *self.flags,
            *sqllogictest_config,
            file,
        ]

        return command


class InputBasedSltRunStepConfig(SltRunStepConfig):
    def __init__(self) -> None:
        super().__init__(file_set=set(), flags=[])

    def configure(self, args: Namespace) -> None:
        if args.pattern is not None:
            file_list = list(file_util.resolve_paths_with_wildcard(args.pattern))
            file_list.sort()

            if len(file_list) == 0:
                raise RuntimeError(f"Pattern {args.pattern} matched no files!")

            print(f"Matched files are: {file_list}")
            self.file_list = file_list

        if args.auto_index_selects:
            self.flags.append("--auto-index-selects")


class InputBasedSltRunConfig(SltRunConfig):
    def __init__(self):
        super().__init__()
        self.steps.append(InputBasedSltRunStepConfig())


class DefaultSltRunStepConfig(SltRunStepConfig):
    def __init__(self, file_set: set[str]):
        super().__init__(
            file_set,
            flags=[],
        )


def compileFastSltConfig() -> SltRunConfig:
    tests = {
        "test/sqllogictest/*.slt",
        "test/sqllogictest/attributes/*.slt",
        "test/sqllogictest/explain/*.slt",
        "test/sqllogictest/introspection/*.slt",
        "test/sqllogictest/autogenerated/*.slt",
        "test/sqllogictest/transform/*.slt",
        "test/sqllogictest/cockroach/aggregate.slt",
        "test/sqllogictest/cockroach/distinct_on.slt",
        "test/sqllogictest/cockroach/subquery_correlated.slt",
        "test/sqllogictest/transform/notice/*.slt",
        "test/sqllogictest/advent-of-code/2023/*.slt",
    }

    # Too slow
    tests_exclude = {
        "test/sqllogictest/default_privileges.slt",
        "test/sqllogictest/distinct_arrangements.slt",
        "test/sqllogictest/privilege_grants.slt",
        "test/sqllogictest/introspection/singlereplica_attribution_sources.slt",
    }

    tests_without_views = {
        "test/sqllogictest/alter.slt",
        "test/sqllogictest/ambiguous_rename.slt",
        "test/sqllogictest/arithmetic.slt",
        "test/sqllogictest/array_fill.slt",
        "test/sqllogictest/arrays.slt",
        "test/sqllogictest/as_of.slt",
        "test/sqllogictest/singlereplica_audit_log.slt",
        "test/sqllogictest/boolean.slt",
        "test/sqllogictest/bytea.slt",
        "test/sqllogictest/cast.slt",
        "test/sqllogictest/char.slt",
        "test/sqllogictest/chbench.slt",
        "test/sqllogictest/chr.slt",
        "test/sqllogictest/cluster.slt",
        "test/sqllogictest/coercion.slt",
        "test/sqllogictest/collate.slt",
        "test/sqllogictest/comparison.slt",
        "test/sqllogictest/cte.slt",
        "test/sqllogictest/cte_lowering.slt",
        "test/sqllogictest/current_database.slt",
        "test/sqllogictest/cursor.slt",
        "test/sqllogictest/datediff.slt",
        "test/sqllogictest/dates-times.slt",
        "test/sqllogictest/degenerate.slt",
        "test/sqllogictest/disambiguate_columns.slt",
        "test/sqllogictest/distinct_from.slt",
        "test/sqllogictest/distinct_on.slt",
        "test/sqllogictest/encode.slt",
        "test/sqllogictest/errors.slt",
        "test/sqllogictest/extract.slt",
        "test/sqllogictest/filter-pushdown.slt",
        "test/sqllogictest/float.slt",
        "test/sqllogictest/funcs.slt",
        "test/sqllogictest/github-14116.slt",
        "test/sqllogictest/github-16036.slt",
        "test/sqllogictest/github-17808.slt",
        "test/sqllogictest/github-2235.slt",
        "test/sqllogictest/github-2327.slt",
        "test/sqllogictest/github-2514.slt",
        "test/sqllogictest/github-2666.slt",
        "test/sqllogictest/github-2668.slt",
        "test/sqllogictest/github-2746.slt",
        "test/sqllogictest/github-2799.slt",
        "test/sqllogictest/github-2906.slt",
        "test/sqllogictest/github-2969.slt",
        "test/sqllogictest/github-2996.slt",
        "test/sqllogictest/github-3256.slt",
        "test/sqllogictest/github-3374.slt",
        "test/sqllogictest/github-3963.slt",
        "test/sqllogictest/github-4336.slt",
        "test/sqllogictest/github-5126.slt",
        "test/sqllogictest/github-5174.slt",
        "test/sqllogictest/github-5474.slt",
        "test/sqllogictest/github-5536.slt",
        "test/sqllogictest/github-5717.slt",
        "test/sqllogictest/github-7585.slt",
        "test/sqllogictest/github-31878.slt",
        "test/sqllogictest/id.slt",
        "test/sqllogictest/id_reuse.slt",
        "test/sqllogictest/information_schema_columns.slt",
        "test/sqllogictest/information_schema_tables.slt",
        "test/sqllogictest/int2vector.slt",
        "test/sqllogictest/interval.slt",
        "test/sqllogictest/joins.slt",
        "test/sqllogictest/jsonb.slt",
        "test/sqllogictest/keys.slt",
        "test/sqllogictest/like.slt",
        "test/sqllogictest/list.slt",
        "test/sqllogictest/list_subquery.slt",
        "test/sqllogictest/managed_cluster.slt",
        "test/sqllogictest/map.slt",
        "test/sqllogictest/materialized_views.slt",
        "test/sqllogictest/mz_catalog_server_index_accounting.slt",
        "test/sqllogictest/mztimestamp.slt",
        "test/sqllogictest/name_resolution.slt",
        "test/sqllogictest/not-null-propagation.slt",
        "test/sqllogictest/numeric.slt",
        "test/sqllogictest/object_ownership.slt",
        "test/sqllogictest/oid.slt",
        "test/sqllogictest/operator.slt",
        "test/sqllogictest/outer_join.slt",
        "test/sqllogictest/outer_join_simplification.slt",
        "test/sqllogictest/parse_ident.slt",
        "test/sqllogictest/pg_catalog_attribute.slt",
        "test/sqllogictest/pg_catalog_class.slt",
        "test/sqllogictest/pg_catalog_matviews.slt",
        "test/sqllogictest/pg_catalog_namespace.slt",
        "test/sqllogictest/pg_catalog_proc.slt",
        "test/sqllogictest/pg_catalog_roles.slt",
        "test/sqllogictest/pg_catalog_tablespace.slt",
        "test/sqllogictest/pg_catalog_views.slt",
        "test/sqllogictest/pg_get_constraintdef.slt",
        "test/sqllogictest/pg_get_indexdef.slt",
        "test/sqllogictest/pg_get_viewdef.slt",
        "test/sqllogictest/pgcli.slt",
        "test/sqllogictest/pgcrypto.slt",
        "test/sqllogictest/postgres-incompatibility.slt",
        "test/sqllogictest/pretty.slt",
        "test/sqllogictest/privilege_checks.slt",
        "test/sqllogictest/privileges_pg.slt",
        "test/sqllogictest/quote_ident.slt",
        "test/sqllogictest/quoting.slt",
        "test/sqllogictest/range.slt",
        "test/sqllogictest/rbac_enabled.slt",
        "test/sqllogictest/record.slt",
        "test/sqllogictest/recursion_limit.slt",
        "test/sqllogictest/recursive_type_unioning.slt",
        "test/sqllogictest/regclass.slt",
        "test/sqllogictest/regproc.slt",
        "test/sqllogictest/regressions.slt",
        "test/sqllogictest/regtype.slt",
        "test/sqllogictest/returning.slt",
        "test/sqllogictest/role.slt",
        "test/sqllogictest/role_create.slt",
        "test/sqllogictest/role_membership.slt",
        "test/sqllogictest/scalar-func-table-position.slt",
        "test/sqllogictest/scalar_subqueries_select_list.slt",
        "test/sqllogictest/schemas.slt",
        "test/sqllogictest/scoping.slt",
        "test/sqllogictest/secret.slt",
        "test/sqllogictest/select_all_group_by.slt",
        "test/sqllogictest/session-window-wmr.slt",
        "test/sqllogictest/show_create_system_objects.slt",
        "test/sqllogictest/slt.slt",
        "test/sqllogictest/source_sizing.slt",
        "test/sqllogictest/string.slt",
        "test/sqllogictest/subquery.slt",
        "test/sqllogictest/subscribe_error.slt",
        "test/sqllogictest/subscribe_outputs.slt",
        "test/sqllogictest/subsource.slt",
        "test/sqllogictest/system-cluster.slt",
        "test/sqllogictest/table_func.slt",
        "test/sqllogictest/temporal.slt",
        "test/sqllogictest/timedomain.slt",
        "test/sqllogictest/timestamp.slt",
        "test/sqllogictest/timestamptz.slt",
        "test/sqllogictest/timezone.slt",
        "test/sqllogictest/tpch_create_index.slt",
        "test/sqllogictest/tpch_create_materialized_view.slt",
        "test/sqllogictest/tpch_select.slt",
        "test/sqllogictest/transactions.slt",
        "test/sqllogictest/type-promotion.slt",
        "test/sqllogictest/typeof.slt",
        "test/sqllogictest/types.slt",
        "test/sqllogictest/uniqueness_propagation_filter.slt",
        "test/sqllogictest/unstable.slt",
        "test/sqllogictest/updates.slt",
        "test/sqllogictest/uuid.slt",
        "test/sqllogictest/vars.slt",
        "test/sqllogictest/web-console.slt",
        "test/sqllogictest/window_funcs.slt",
        "test/sqllogictest/attributes/mir_arity.slt",
        "test/sqllogictest/attributes/mir_column_types.slt",
        "test/sqllogictest/attributes/mir_unique_keys.slt",
        "test/sqllogictest/explain/bad_explain_statements.slt",
        "test/sqllogictest/explain/decorrelated_plan_as_json.slt",
        "test/sqllogictest/explain/decorrelated_plan_as_text.slt",
        "test/sqllogictest/explain/optimized_plan_as_json.slt",
        "test/sqllogictest/explain/optimized_plan_as_text.slt",
        "test/sqllogictest/explain/physical_plan_as_json.slt",
        "test/sqllogictest/explain/physical_plan_as_text.slt",
        "test/sqllogictest/explain/raw_plan_as_json.slt",
        "test/sqllogictest/explain/raw_plan_as_text.slt",
        "test/sqllogictest/autogenerated/all_parts_essential.slt",
        "test/sqllogictest/autogenerated/char-varchar-comparisons.slt",
        "test/sqllogictest/transform/aggregation_nullability.slt",
        "test/sqllogictest/transform/column_knowledge.slt",
        "test/sqllogictest/transform/dataflow.slt",
        "test/sqllogictest/transform/demand.slt",
        "test/sqllogictest/transform/filter_index.slt",
        "test/sqllogictest/transform/fold_constants.slt",
        "test/sqllogictest/transform/is_null_propagation.slt",
        "test/sqllogictest/transform/join_fusion.slt",
        "test/sqllogictest/transform/join_index.slt",
        "test/sqllogictest/transform/literal_constraints.slt",
        "test/sqllogictest/transform/literal_lifting.slt",
        "test/sqllogictest/transform/monotonic.slt",
        "test/sqllogictest/transform/non_null_requirements.slt",
        "test/sqllogictest/transform/normalize_lets.slt",
        "test/sqllogictest/transform/predicate_pushdown.slt",
        "test/sqllogictest/transform/predicate_reduction.slt",
        "test/sqllogictest/transform/projection_lifting.slt",
        "test/sqllogictest/transform/reduce_elision.slt",
        "test/sqllogictest/transform/reduce_fusion.slt",
        "test/sqllogictest/transform/reduction_pushdown.slt",
        "test/sqllogictest/transform/redundant_join.slt",
        "test/sqllogictest/transform/relation_cse.slt",
        "test/sqllogictest/transform/scalability.slt",
        "test/sqllogictest/transform/scalar_cse.slt",
        "test/sqllogictest/transform/threshold_elision.slt",
        "test/sqllogictest/transform/topk.slt",
        "test/sqllogictest/transform/union.slt",
        "test/sqllogictest/transform/union_cancel.slt",
        # "test/sqllogictest/sqlite/test/evidence/in1.test",
        "test/sqllogictest/sqlite/test/evidence/in2.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_aggfunc.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_createtrigger.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_createview.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_dropindex.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_droptable.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_droptrigger.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_dropview.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_reindex.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_replace.test",
        "test/sqllogictest/sqlite/test/evidence/slt_lang_update.test",
        # TODO(benesch): renable a fast subset of the following when performance is restored.
        # "test/sqllogictest/sqlite/test/index/between/1/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/commute/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/delete/1/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/in/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/orderby_nosort/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/orderby/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/random/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/index/view/10/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/random/aggregates/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/random/expr/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/random/groupby/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/random/select/slt_good_0.test",
        # "test/sqllogictest/sqlite/test/select1.test",
        # "test/sqllogictest/sqlite/test/select2.test",
        # "test/sqllogictest/sqlite/test/select3.test",
        # "test/sqllogictest/sqlite/test/select4.test",
        # "test/sqllogictest/sqlite/test/select5.test",
        "test/sqllogictest/cockroach/alias_types.slt",
        "test/sqllogictest/cockroach/alter_column_type.slt",
        "test/sqllogictest/cockroach/alter_table.slt",
        "test/sqllogictest/cockroach/apply_join.slt",
        "test/sqllogictest/cockroach/array.slt",
        "test/sqllogictest/cockroach/as_of.slt",
        "test/sqllogictest/cockroach/bit.slt",
        # "test/sqllogictest/cockroach/builtin_function.slt",
        "test/sqllogictest/cockroach/bytes.slt",
        "test/sqllogictest/cockroach/case_sensitive_names.slt",
        # "test/sqllogictest/cockroach/collatedstring_constraint.slt",
        "test/sqllogictest/cockroach/collatedstring_index1.slt",
        "test/sqllogictest/cockroach/collatedstring_index2.slt",
        "test/sqllogictest/cockroach/collatedstring_normalization.slt",
        "test/sqllogictest/cockroach/collatedstring_nullinindex.slt",
        "test/sqllogictest/cockroach/collatedstring_uniqueindex1.slt",
        "test/sqllogictest/cockroach/collatedstring_uniqueindex2.slt",
        "test/sqllogictest/cockroach/collatedstring.slt",
        "test/sqllogictest/cockroach/computed.slt",
        # "test/sqllogictest/cockroach/conditional.slt",
        "test/sqllogictest/cockroach/create_as.slt",
        "test/sqllogictest/cockroach/custom_escape_character.slt",
        "test/sqllogictest/cockroach/database.slt",
        # "test/sqllogictest/cockroach/datetime.slt",
        # "test/sqllogictest/cockroach/decimal.slt",
        "test/sqllogictest/cockroach/delete.slt",
        "test/sqllogictest/cockroach/discard.slt",
        "test/sqllogictest/cockroach/drop_database.slt",
        "test/sqllogictest/cockroach/drop_table.slt",
        "test/sqllogictest/cockroach/drop_user.slt",
        "test/sqllogictest/cockroach/drop_view.slt",
        "test/sqllogictest/cockroach/errors.slt",
        # "test/sqllogictest/cockroach/exec_hash_join.slt",
        # "test/sqllogictest/cockroach/exec_merge_join.slt",
        "test/sqllogictest/cockroach/exec_window.slt",
        "test/sqllogictest/cockroach/extract.slt",
        # "test/sqllogictest/cockroach/float.slt",
        "test/sqllogictest/cockroach/inet.slt",
        "test/sqllogictest/cockroach/information_schema.slt",
        "test/sqllogictest/cockroach/insert.slt",
        "test/sqllogictest/cockroach/int_size.slt",
        "test/sqllogictest/cockroach/join.slt",
        # "test/sqllogictest/cockroach/json_builtins.slt",
        # "test/sqllogictest/cockroach/json.slt",
        "test/sqllogictest/cockroach/like.slt",
        "test/sqllogictest/cockroach/limit.slt",
        # "test/sqllogictest/cockroach/lookup_join.slt",
        "test/sqllogictest/cockroach/namespace.slt",
        # "test/sqllogictest/cockroach/no_primary_key.slt",
        "test/sqllogictest/cockroach/order_by.slt",
        "test/sqllogictest/cockroach/ordinal_references.slt",
        "test/sqllogictest/cockroach/ordinality.slt",
        "test/sqllogictest/cockroach/orms-opt.slt",
        "test/sqllogictest/cockroach/orms.slt",
        "test/sqllogictest/cockroach/pg_catalog.slt",
        "test/sqllogictest/cockroach/pgoidtype.slt",
        # "test/sqllogictest/cockroach/postgres_jsonb.slt",
        # "test/sqllogictest/cockroach/postgresjoin.slt",
        "test/sqllogictest/cockroach/prepare.slt",
        "test/sqllogictest/cockroach/rename_column.slt",
        "test/sqllogictest/cockroach/rename_constraint.slt",
        "test/sqllogictest/cockroach/rename_database.slt",
        "test/sqllogictest/cockroach/rename_table.slt",
        "test/sqllogictest/cockroach/rename_view.slt",
        "test/sqllogictest/cockroach/returning.slt",
        "test/sqllogictest/cockroach/rows_from.slt",
        "test/sqllogictest/cockroach/scale.slt",
        "test/sqllogictest/cockroach/select_index_flags.slt",
        # "test/sqllogictest/cockroach/select_index_span_ranges.slt",
        # "test/sqllogictest/cockroach/select_index.slt",
        "test/sqllogictest/cockroach/select_search_path.slt",
        "test/sqllogictest/cockroach/select_table_alias.slt",
        # "test/sqllogictest/cockroach/select.slt",
        "test/sqllogictest/cockroach/shift.slt",
        # "test/sqllogictest/cockroach/sqlsmith.slt",
        "test/sqllogictest/cockroach/srfs.slt",
        # "test/sqllogictest/cockroach/statement_source.slt",
        # "test/sqllogictest/cockroach/suboperators.slt",
        # "test/sqllogictest/cockroach/subquery-opt.slt",
        "test/sqllogictest/cockroach/subquery.slt",
        "test/sqllogictest/cockroach/table.slt",
        # "test/sqllogictest/cockroach/target_names.slt",
        # "test/sqllogictest/cockroach/time.slt",
        "test/sqllogictest/cockroach/timestamp.slt",
        "test/sqllogictest/cockroach/truncate.slt",
        "test/sqllogictest/cockroach/tuple.slt",
        # "test/sqllogictest/cockroach/typing.slt",
        # "test/sqllogictest/cockroach/union-opt.slt",
        "test/sqllogictest/cockroach/union.slt",
        "test/sqllogictest/cockroach/update.slt",
        "test/sqllogictest/cockroach/upsert.slt",
        "test/sqllogictest/cockroach/uuid.slt",
        "test/sqllogictest/cockroach/values.slt",
        # "test/sqllogictest/cockroach/views.slt",
        "test/sqllogictest/cockroach/where.slt",
        "test/sqllogictest/cockroach/window.slt",
        "test/sqllogictest/cockroach/with.slt",
        # "test/sqllogictest/cockroach/zero.slt",
        "test/sqllogictest/postgres/float4.slt",
        "test/sqllogictest/postgres/float8.slt",
        "test/sqllogictest/postgres/join-lateral.slt",
        "test/sqllogictest/postgres/regex.slt",
        "test/sqllogictest/postgres/subselect.slt",
        "test/sqllogictest/postgres/pgcrypto/*.slt",
        "test/sqllogictest/introspection/cluster_log_compaction.slt",
    }

    tests = file_util.resolve_paths_with_wildcard(tests)
    tests_without_views = file_util.resolve_paths_with_wildcard(tests_without_views)
    tests_with_views = tests - tests_without_views - tests_exclude

    config = SltRunConfig()
    config.steps.append(DefaultSltRunStepConfig(tests_without_views))
    config.steps.append(SltRunStepConfig(tests_with_views, ["--auto-index-selects"]))
    return config


def compileSlowSltConfig() -> SltRunConfig:
    config = SltRunConfig()

    # All CockroachDB and PostgreSQL SLTs can be run with --auto-index-selects, but require --no-fail
    cockroach_and_pg_tests = {
        "test/sqllogictest/cockroach/*.slt",
        "test/sqllogictest/postgres/*.slt",
        "test/sqllogictest/postgres/pgcrypto/*.slt",
    }
    cockroach_and_pg_tests = file_util.resolve_paths_with_wildcard(
        cockroach_and_pg_tests
    )

    config.steps.append(
        SltRunStepConfig(cockroach_and_pg_tests, ["--auto-index-selects", "--no-fail"])
    )

    tests = {
        "test/sqllogictest/*.slt",
        "test/sqllogictest/attributes/*.slt",
        "test/sqllogictest/introspection/*.slt",
        "test/sqllogictest/explain/*.slt",
        "test/sqllogictest/transform/*.slt",
        "test/sqllogictest/transform/fold_vs_dataflow/*.slt",
        "test/sqllogictest/transform/notice/*.slt",
        "test/sqllogictest/special/*",
    }
    tests_without_views_and_replica = {
        # errors:
        # https://github.com/MaterializeInc/database-issues/issues/6181
        "test/sqllogictest/list.slt",
        # transactions:
        "test/sqllogictest/distinct_arrangements.slt",
        "test/sqllogictest/github-3374.slt",
        "test/sqllogictest/introspection/cluster_log_compaction.slt",
        "test/sqllogictest/introspection/singlereplica_attribution_sources.slt",
        "test/sqllogictest/timedomain.slt",
        "test/sqllogictest/transactions.slt",
        # depends on unmaterializable functions
        "test/sqllogictest/regclass.slt",
        "test/sqllogictest/regproc.slt",
        "test/sqllogictest/regtype.slt",
        # different outputs:
        # seems expected for audit log to be different
        "test/sqllogictest/singlereplica_audit_log.slt",
        # different indexes auto-created
        "test/sqllogictest/cluster.slt",
        # different indexes auto-created
        "test/sqllogictest/object_ownership.slt",
        # https://github.com/MaterializeInc/database-issues/issues/6011
        "test/sqllogictest/interval.slt",
        # https://github.com/MaterializeInc/database-issues/issues/6011
        "test/sqllogictest/operator.slt",
        # specific replica size tested:
        "test/sqllogictest/managed_cluster.slt",
        "test/sqllogictest/web-console.slt",
        "test/sqllogictest/show_clusters.slt",
    }
    tests_no_auto_index_selects = {
        # pg_typeof contains public schema name in views
        "test/sqllogictest/cast.slt",
        "test/sqllogictest/map.slt",
        # pg_typeof contains public schema name in views
        "test/sqllogictest/typeof.slt",
        # https://github.com/MaterializeInc/database-issues/issues/9513#issuecomment-3128051157
        "test/sqllogictest/temporal.slt",
    }

    tests = file_util.resolve_paths_with_wildcard(tests)
    tests_without_views_and_replica = file_util.resolve_paths_with_wildcard(
        tests_without_views_and_replica
    )
    tests_no_auto_index_selects = file_util.resolve_paths_with_wildcard(
        tests_no_auto_index_selects
    )
    tests_with_views_or_replica = (
        tests - tests_without_views_and_replica - tests_no_auto_index_selects
    )

    config.steps.append(
        SltRunStepConfig(tests_with_views_or_replica, ["--auto-index-selects"])
    )
    config.steps.append(
        SltRunStepConfig(
            (tests_without_views_and_replica | tests_no_auto_index_selects), []
        )
    )

    # Due to performance issues (see below), we pick two selected SLTs from
    # the SQLite corpus that we can reasonably run with --auto-index-selects
    # and that include min/max query patterns. Note that none of the SLTs in
    # the corpus we presently use from SQLite contain top-k patterns.
    tests_with_views = {
        "test/sqllogictest/sqlite/test/index/random/1000/slt_good_0.test",
        "test/sqllogictest/sqlite/test/random/aggregates/slt_good_129.test",
    }

    tests = set(file_util.get_recursive_file_list("test/sqllogictest/sqlite/test"))
    tests_without_views = tests - tests_with_views

    # Run selected tests with --auto-index-selects
    config.steps.append(
        SltRunStepConfig(
            tests_with_views,
            ["--auto-index-selects", "--enable-table-keys"],
        )
    )

    # Too slow to run with --auto-index-selects, can't run together with
    # --auto-transactions, no differences seen in previous run. We might want to
    # revisit and see if we can periodically test them, even if it takes 2-3 days
    # for the run to finish.
    config.steps.append(
        SltRunStepConfig(
            tests_without_views,
            ["--auto-transactions", "--auto-index-tables", "--enable-table-keys"],
        )
    )

    return config