module TestRunner

using CTBase: CTBase

using DocStringExtensions

using Test: Test, @testset

const TestSpec = Union{Symbol,String}

struct TestRunInfo

spec::TestSpec

filename::String

func_symbol::Union{Symbol,Nothing}

index::Int

total::Int

status::Symbol

error::Union{Exception,Nothing}

elapsed::Union{Float64,Nothing}

function CTBase.run_tests(

::CTBase.Extensions.TestRunnerTag;

args::AbstractVector{<:AbstractString}=String[],

testset_name::String="Tests",

available_tests=Symbol[],

filename_builder::Function=identity,

funcname_builder::Function=identity,

eval_mode::Bool=true,

verbose::Bool=true,

showtiming::Bool=true,

test_dir::String=joinpath(pwd(), "test"),

on_test_start::Union{Function,Nothing}=nothing,

on_test_done::Union{Function,Nothing}=nothing,

progress::Bool=true,

)

# Guard: a subdirectory named "test" inside test_dir would conflict with

# the automatic `test/` prefix stripping in _parse_test_args.

if isdir(joinpath(test_dir, "test"))

error(

"A subdirectory \"test\" exists inside the test directory " *

"\"$(test_dir)\". This is not supported because selection " *

"arguments starting with \"test/\" are automatically stripped " *

"(e.g. \"test/suite\" → \"suite\"). Please rename the subdirectory.",

)

end

# Parse command-line arguments

(selections, run_all, dry_run) = _parse_test_args(String.(args))

available_tests_vec = _normalize_available_tests(available_tests)

# Get selected tests

selected = _select_tests(

selections, available_tests_vec, run_all, filename_builder; test_dir=test_dir

)

if dry_run

println("Dry run: the following tests would be executed:")

println(join(selected, "\n"))

return nothing

end

total = length(selected)

# Wire up default progress callback when no custom on_test_done is provided

effective_on_test_done = if on_test_done !== nothing

on_test_done

elseif progress

_make_default_on_test_done(stdout, total)

else

nothing

end

@testset verbose = verbose showtiming = showtiming "$testset_name" begin

for (idx, spec) in enumerate(selected)

@testset "$(spec)" begin

_run_single_test(

spec;

available_tests=available_tests_vec,

filename_builder,

funcname_builder,

eval_mode,

test_dir,

index=idx,

total,

on_test_start,

on_test_done=effective_on_test_done,

)

end

end

end

function _parse_test_args(args::Vector{String})

selections = String[]

run_all = false

dry_run = false

for arg in args

if arg in ("coverage=true", "coverage", "--coverage", "coverage=false")

continue

elseif arg == "-a" || arg == "--all"

run_all = true

elseif arg == "-n" || arg == "--dryrun"

dry_run = true

else

push!(selections, _strip_test_prefix(arg))

end

end

return (selections, run_all, dry_run)

function _strip_test_prefix(s::AbstractString)

for prefix in ("test/", "test\\")

if startswith(s, prefix)

return String(s[(length(prefix) + 1):end])

end

end

return String(s)

function _normalize_selections(selections::Vector{String}, candidates::Vector{<:TestSpec})

candidate_strs = [c isa Symbol ? String(c) : String(c) for c in candidates]

normalized = String[]

for sel in selections

# Strip trailing slash(es)

s = rstrip(sel, '/')

push!(normalized, s)

# If no glob wildcard, check if it looks like a directory prefix

if !occursin('*', s) && !occursin('?', s)

prefix = s * "/"

is_dir_prefix = any(c -> startswith(c, prefix), candidate_strs)

if is_dir_prefix

push!(normalized, s * "/*")

end

end

end

return unique(normalized)

function _glob_to_regex(pattern::AbstractString)

# Escape special regex characters except * and ?

regex_str = replace(

pattern,

"." => "\\.",

"+" => "\\+",

"(" => "\\(",

")" => "\\)",

"[" => "\\[",

"]" => "\\]",

"{" => "\\{",

"}" => "\\}",

"^" => "\\^",

"\u0024" => "\\\u0024",

"\\" => "\\\\",

)

# Convert glob wildcards to regex wildcards

regex_str = replace(regex_str, "*" => ".*", "?" => ".")

# Anchor to full string

return Regex("^" * regex_str * "\u0024")

function _ensure_jl(filename::AbstractString)

endswith(filename, ".jl") ? filename : filename * ".jl"

function _builder_to_string(x)

x isa Symbol ? String(x) : String(x)

function _normalize_available_tests(available_tests)

available_tests === nothing && return TestSpec[]

if !(available_tests isa AbstractVector || available_tests isa Tuple)

throw(ArgumentError("available_tests must be a Vector or Tuple of Symbol/String"))

end

out = TestSpec[]

for entry in available_tests

if entry isa Symbol || entry isa String

push!(out, entry)

else

throw(ArgumentError("available_tests entries must be Symbol or String"))

end

end

return out

function _collect_test_files_recursive(test_dir::AbstractString)

files = String[]

for (root, _, fs) in walkdir(test_dir)

for f in fs

if endswith(f, ".jl") && f != "runtests.jl"

full = joinpath(root, f)

push!(files, relpath(full, test_dir))

end

end

end

sort!(files)

return files

function _find_symbol_test_file_rel(

name::Symbol, filename_builder::Function; test_dir::AbstractString

)

wanted = _ensure_jl(_builder_to_string(filename_builder(name)))

all = _collect_test_files_recursive(test_dir)

matches = filter(f -> basename(f) == wanted, all)

if isempty(matches)

return nothing

end

if wanted in matches

return wanted

end

sort!(matches; by=f -> (count(==('/'), f), ncodeunits(f), f))

return first(matches)

function _select_tests(

selections::Vector{String},

available_tests::AbstractVector{<:TestSpec},

run_all::Bool,

filename_builder::Function;

test_dir::String=joinpath(pwd(), "test"), # Default assumption

)

# 1. Identify all potential test files

# We look for .jl files in test_dir, excluding runtests.jl

all_files = filter(f -> endswith(f, ".jl") && f != "runtests.jl", readdir(test_dir))

# Map filenames back to "test names" is tricky without reverse builder.

# Strategy:

# We assume `available_tests` defines the canonical list of "names".

# If `available_tests` is empty, we derive names from files?

# -> User said: "list all .jl files... but only keep available if provided"

candidates = TestSpec[]

if isempty(available_tests)

# If no available_tests provided, every .jl file is a candidate!

# This effectively AUTO-DISCOVERS tests.

# We need to guess the "name" from the filename.

# This is hard because filename_builder is name -> filename.

# utils -> test_utils.jl

# test_utils.jl -> utils ??

# For now, let's keep the user's current behavior:

# If available_tests is empty, the logic relies on what the user passes.

# BUT the new logic says "if args empty -> run all".

# So we MUST perform auto-discovery if we want "run all" to work without explicit available_tests.

# Heuristic: if file starts with "test_", strip it?

# Or just use the basename as the symbol?

# Let's assume the "name" is the filename without extension for auto-discovery?

# Or better: don't guess names yet. Just work with filenames?

# But `run_tests` iterates over `names`.

# Let's look at existing files: test_utils.jl -> name=:utils (via filename_builder)

# We cannot invert `filename_builder` (F -> S).

# So we are stuck if we want to infer names from files.

# COMPROMISE: If available_tests is empty, we cannot guarantee auto-discovery compatibility with arbitrary filename_builders.

# We will assume a default mapping for auto-discovery: name = file_basename_without_extension.

for f in all_files

name_str = replace(f, ".jl" => "")

# If name starts with "test_", removing it is common convention, but maybe risky?

# Let's keep the full basename as the name if we are auto-discovering.

push!(candidates, Symbol(name_str))

end

else

# If available_tests IS provided, we only consider these.

# We verify if their files exist.

recursive_files = _collect_test_files_recursive(test_dir)

for entry in available_tests

if entry isa Symbol

rel = _find_symbol_test_file_rel(entry, filename_builder; test_dir=test_dir)

if rel !== nothing

push!(candidates, entry)

end

else

full = joinpath(test_dir, entry)

if isdir(full)

prefix = entry * "/"

for f in recursive_files

if startswith(f, prefix)

push!(candidates, f)

end

end

else

regex = _glob_to_regex(entry)

for f in recursive_files

f_no_ext = replace(f, ".jl" => "")

if !isnothing(match(regex, f)) || !isnothing(match(regex, f_no_ext))

push!(candidates, f)

end

end

end

end

end

end

# If run_all is requested or no selections, return all candidates

if run_all || isempty(selections)

return candidates

end

# 3. Normalize selections: expand bare directory paths to dir/*

selections = _normalize_selections(selections, candidates)

# 4. Filter candidates by selections (Patterns)

filtered = TestSpec[]

for candidate in candidates

candidate_str = candidate isa Symbol ? String(candidate) : String(candidate)

# Also check the associated filename?

# If I have candidate :utils -> test_utils.jl

# And user passes "test_u*", it should match "test_utils.jl" OR "utils"?

# User said "Scan test/ directory... ARGS are globs".

# So matching against the FILENAME seems primary.

# Resolve filename for candidate

if candidate isa String

candidate_filename = _ensure_jl(candidate)

elseif isempty(available_tests)

candidate_filename = "$(candidate).jl"

else

candidate_filename = _ensure_jl(_builder_to_string(filename_builder(candidate)))

end

# Also match strictly against filename without extension?

candidate_filename_no_ext = replace(candidate_filename, ".jl" => "")

candidate_basename = basename(candidate_filename)

candidate_basename_no_ext = replace(candidate_basename, ".jl" => "")

candidate_basename_no_test_prefix =

if startswith(candidate_basename_no_ext, "test_")

candidate_basename_no_ext[6:end]

else

candidate_basename_no_ext

end

matched = false

for sel in selections

regex = _glob_to_regex(sel)

# Match against:

# 1. Candidate name (e.g. :utils)

# 2. Filename (e.g. test_utils.jl)

# 3. Filename without extension (e.g. test_utils)

if !isnothing(match(regex, candidate_str)) ||

!isnothing(match(regex, candidate_filename)) ||

!isnothing(match(regex, candidate_filename_no_ext)) ||

!isnothing(match(regex, candidate_basename)) ||

!isnothing(match(regex, candidate_basename_no_ext)) ||

!isnothing(match(regex, candidate_basename_no_test_prefix))

matched = true

break

end

end

if matched

push!(filtered, candidate)

end

end

return filtered

function _run_single_test(

spec::TestSpec;

available_tests::AbstractVector{<:TestSpec},

filename_builder::Function,

funcname_builder::Function,

eval_mode::Bool,

test_dir::String,

index::Int=1,

total::Int=1,

on_test_start::Union{Function,Nothing}=nothing,

on_test_done::Union{Function,Nothing}=nothing,

)

# --- Resolve filename and func_symbol ---

filename, func_symbol = _resolve_test(

spec; available_tests, filename_builder, funcname_builder, eval_mode, test_dir

)

# --- Include the file ---

Base.include(Main, filename)

# --- Check function exists after include ---

if eval_mode && func_symbol !== nothing && !isdefined(Main, func_symbol)

error("""

""")

end

# --- on_test_start callback ---

if on_test_start !== nothing

info = TestRunInfo(

spec, filename, func_symbol, index, total, :pre_eval, nothing, nothing

)

should_continue = on_test_start(info)

if should_continue === false

if on_test_done !== nothing

done_info = TestRunInfo(

spec, filename, func_symbol, index, total, :skipped, nothing, nothing

)

on_test_done(done_info)

end

return nothing

end

end

# --- Skip eval if not in eval mode ---

if !eval_mode || func_symbol === nothing

if on_test_done !== nothing

done_info = TestRunInfo(

spec, filename, func_symbol, index, total, :skipped, nothing, nothing

)

on_test_done(done_info)

end

return nothing

end

# --- Snapshot testset results before eval ---

ts = Test.get_testset()

n_before = (ts isa Test.DefaultTestSet) ? length(ts.results) : nothing

# --- Eval the function ---

t0 = time()

try

Main.eval(Expr(:call, func_symbol))

elapsed = time() - t0

if on_test_done !== nothing

# Detect @test failures by scanning only the new results added during eval

status = if n_before !== nothing

_has_failures_in_results(ts, n_before + 1) ? :test_failed : :post_eval

else

:post_eval

end

done_info = TestRunInfo(

spec, filename, func_symbol, index, total, status, nothing, elapsed

)

on_test_done(done_info)

end

catch ex

elapsed = time() - t0

if on_test_done !== nothing

done_info = TestRunInfo(

spec, filename, func_symbol, index, total, :error, ex, elapsed

)

on_test_done(done_info)

end

rethrow()

end

return nothing

function _resolve_test(

spec::TestSpec;

available_tests::AbstractVector{<:TestSpec},

filename_builder::Function,

funcname_builder::Function,

eval_mode::Bool,

test_dir::String,

)

if spec isa String

rel = _ensure_jl(spec)

filename = joinpath(test_dir, rel)

if !isfile(filename)

error("""

""")

end

func_symbol = if eval_mode

Symbol(replace(basename(rel), ".jl" => ""))

else

nothing

end

return (filename, func_symbol)

end

name = spec

# Build filename

rel = _find_symbol_test_file_rel(name, filename_builder; test_dir=test_dir)

rel === nothing && error("""

""")

filename = joinpath(test_dir, rel)

# Check file exists

!isfile(filename) && error("""

""")

# Determine function name

func_symbol = funcname_builder(name)

# Check consistency: eval_mode=true but funcname_builder returns nothing

if eval_mode && func_symbol === nothing

error(

"""

""",

)

end

if !eval_mode || func_symbol === nothing

return (filename, nothing)

end

func_symbol = func_symbol isa Symbol ? func_symbol : Symbol(String(func_symbol))

return (filename, func_symbol)

const _FULL_BAR_THRESHOLD = 50