Skip to content

add benchmark and test commands to the standard library #8696

@amtoine

Description

@amtoine

Related problem

related to a point in #8311 about benchmarking and a #cool-script

Describe the solution you'd like

def "from ns" [] {
    [$in "ns"] | str join | into duration
}

def benchmark [
    code: closure
    -n: int = 50
    --verbose (-v): bool
    --pretty: bool
] {
    let times = (
        seq 1 $n
        | each {|i|
            if $verbose { print -n $"($i) / ($n)\r" }
            timeit { do $code } | into int | into decimal
        }
    )

    if $verbose { print $"($n) / ($n)" }

    let report = {
        mean: ($times | math avg | from ns)
        std: ($times | math stddev | from ns)
        times: ($times | each { from ns })
    }

    if $pretty {
        $"($report.mean) +/- ($report.std)"
    } else {
        $report
    }
}
  • testing
# convert an integer amount of nanoseconds into a duration
#
# # Examples
# ```nushell
# use std.nu
#
# std assert eq (123 | from ns) 123ns
# std assert eq (987654321 | from ns) (987ms + 654us + 321ns)
# ```
# should not return any error
def "from ns" [] {
    [$in "ns"] | str join | into duration
}

# Test-driven development for nushell commands
#
# Examples:
#     >_ femtotest { 1 == 1 } --expect true | reject bench.times
#     ╭────────┬────────────────────────╮
#     │ source │ { 1 == 1 }             │
#     │ expect │ true                   │
#     │ output │ true                   │
#     │        │ ╭────────┬───────────╮ │
#     │ bench  │ │ avg    │ 5µs 895ns │ │
#     │        │ │ stddev │ 7µs 204ns │ │
#     │        │ ╰────────┴───────────╯ │
#     │ pass   │ true                   │
#     ╰────────┴────────────────────────╯
#
#     >_ femtotest { 1 == 1 } --expect false | reject bench.times
#     ╭────────┬────────────────────────╮
#     │ source │ { 1 == 1 }             │
#     │ expect │ false                  │
#     │ output │ true                   │
#     │        │ ╭────────┬───────────╮ │
#     │ bench  │ │ avg    │ 6µs 208ns │ │
#     │        │ │ stddev │ 8µs 800ns │ │
#     │        │ ╰────────┴───────────╯ │
#     │ pass   │ false                  │
#     ╰────────┴────────────────────────╯
#
#     >_ femtotest { 'Hello world!' } --expect 'foobar' --rounds 1000 | reject bench.times
#     ╭────────┬────────────────────────╮
#     │ source │ { 'Hello world!' }     │
#     │ expect │ foobar                 │
#     │ output │ Hello world!           │
#     │        │ ╭────────┬───────────╮ │
#     │ bench  │ │ avg    │ 3µs 695ns │ │
#     │        │ │ stddev │ 2µs 328ns │ │
#     │        │ ╰────────┴───────────╯ │
#     │ pass   │ false                  │
#     ╰────────┴────────────────────────╯
export def femtotest [
    target: closure         # Testing target closure, i.e. the command to test
    --expect (-e): any      # The expected value to obtain from the target command
    --rounds: int = 100     # Number of rounds to benchmark target execution times
    ] {
    let output = (do $target)

    let times = (0..$rounds | par-each {||
        timeit { do $target }
    })

    {
        source: (view source $target | nu-highlight)
        expect: $expect
        output: $output
        bench: {
            times: $times
            avg: ($times | into int | into decimal | math avg | from ns)
            stddev: ($times | into int | into decimal | math stddev | from ns)
        }
        pass: ($expect == $output)
    }
}

# Self test the femtotest command
export def "femtotest test" [] {
    let testchain = [
        {
            exec: {|| 1 == 1 }
            expect: true
        }
        {
            exec: {|| 1 == 1 }
            expect: false
        }
    ]

    $testchain | par-each {
        |unit| femtotest $unit.exec --expect $unit.expect
    }
}

Describe alternatives you've considered

No response

Additional context and details

No response

Metadata

Metadata

Assignees

No one assigned

    Labels

    A:std-libraryDefining and improving the standard library written in Nucategory:enhancementNew feature or request

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions