PageRenderTime 44ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/src/RandomizedPropertyTest.jl

https://git.sr.ht/~quf/RandomizedPropertyTest.jl/
Julia | 443 lines | 319 code | 98 blank | 26 comment | 78 complexity | 52c9f12fc6f756d49c6ceb1f77d4a0e4 MD5 | raw file
Possible License(s): GPL-3.0
  1. #=
  2. This file is part of the RandomizedPropertyTest.jl project.
  3. Copyright © 2019 Lukas Himbert
  4. RandomizedPropertyTest.jl is free software:
  5. you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License.
  6. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  7. without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  8. See the GNU General Public License for more details.
  9. You should have received a copy of the GNU General Public License along with this program.
  10. If not, see <https://www.gnu.org/licenses/>.
  11. =#
  12. """
  13. RandomizedPropertyTest
  14. Performs randomized property tests (also known as specification tests) of your programs or functions.
  15. For usage information, see `@quickcheck`.
  16. """
  17. module RandomizedPropertyTest
  18. using Random: MersenneTwister, AbstractRNG, randexp
  19. import Base.product
  20. import Base.Iterators.flatten
  21. using Logging: @warn
  22. export @quickcheck
  23. """
  24. @quickcheck [n=nexpr] expr (vartuple :: T) [...]
  25. Check whether the property expressed by `expr` holds for variables `vartuple...` of type `T`.
  26. Multiple such variable declarations may be given.
  27. Returns `false` if a counterexample was found, `true` otherwise.
  28. It should be noted that a result of `true` does not constitute proof of the property.
  29. `nexpr` is the number of pseudorandom inputs used to examine the veracity of the property.
  30. It has no effect on special cases, which are always checked.
  31. To check only special cases, you may set `nexpr` to zero.
  32. For reproducibility of counterexamples, inputs are chosen pseudorandomly with a fixed seed.
  33. Instead of running `@quickcheck` multiple times to be more certain of the property you wish to verify, run it once with a larger `n`.
  34. To use `@quickcheck` with custom data types or custom distributions of builtin datatypes, see `generate` and `specialcases`.
  35. Some data types for custom distributions (e.g. `Range{T,a,b}`) are predefined in this module.
  36. Examples
  37. --------
  38. Check the associativity of `+` for Ints:
  39. ```jldoctest
  40. julia> @quickcheck (a+b == b+a) (a :: Int) (b :: Int)
  41. true
  42. ```
  43. The same test with alternative syntax and a larger number of tests:
  44. ```jldoctest
  45. julia> @quickcheck n=10^6 (a+b == b+a) ((a, b) :: Int)
  46. true
  47. ```
  48. On the other hand, a test of the associativity of double-precision floats fails, even if only finite values are allowed (no `NaN`, ±`Inf`):
  49. ```jldoctest
  50. julia> @quickcheck (a+(b+c) == (a+b)+c || !all(isfinite, (a,b,c))) ((a,b,c) :: Float64)
  51. ┌ Warning: Property `a + (b + c) == (a + b) + c || (any(isnan, (a, b, c)) || any(isinf, (a, b, c)))` does not hold for (a = 0.3333333333333333, b = 1.0, c = 1.0).
  52. └ @ RandomizedPropertyTest ~/store/zeug/public/RandomizedPropertyTest/src/RandomizedPropertyTest.jl:119
  53. false
  54. ```
  55. Test an addition theorem of `sin` over one period:
  56. ```jldoctest
  57. julia> @quickcheck (sin(α - β) ≈ sin(α) * cos(β) - cos(α) * sin(β)) ((α, β) :: Range{Float64, 0, 2π})
  58. true
  59. ```
  60. """
  61. macro quickcheck(args...)
  62. names = Symbol[]
  63. types = []
  64. length(args) >= 2 || error("Use as @quickcheck [n=nexpr] expr type [...]")
  65. if args[1] isa Expr && args[1].head == :(=) && args[1].args[1] == :n
  66. nexpr = esc(args[1].args[2])
  67. args = args[2:length(args)]
  68. length(args) >= 2 || error("Use as @quickcheck [n=nexpr] expr type [...]")
  69. else
  70. nexpr = 10^4
  71. end
  72. expr = args[1]
  73. vartypes = args[2:length(args)]
  74. length(vartypes) > 0 || error("No variable declared. Please use @test to test properties with no free variables.")
  75. for e in vartypes
  76. e isa Expr && e.head == :(::) || error("Invalid variable declaration `$e`.")
  77. if e.args[1] isa Symbol
  78. newsymbols = Symbol[e.args[1]]
  79. elseif e.args[1].head == :tuple && all(x->x isa Symbol, e.args[1].args)
  80. newsymbols = e.args[1].args
  81. else
  82. error("Invalid variable declaration `$e`.")
  83. end
  84. all(x -> !(x in names), newsymbols) || error("Duplicate declaration of $(e.args[1]).")
  85. for symb in newsymbols
  86. push!(names, symb)
  87. push!(types, e.args[2])
  88. end
  89. end
  90. nametuple = Expr(:tuple, names...)
  91. typetuple = esc(Expr(:tuple, types...)) # escaping is required for user-provided types
  92. exprstr = let io = IOBuffer(); print(io, expr); seek(io, 0); read(io, String); end
  93. namestrs = [String(n) for n in names]
  94. fexpr = esc(Expr(:(->), nametuple, expr)) # escaping is required for global (and other) variables in the calling scope
  95. return quote
  96. do_quickcheck($fexpr, $exprstr, $namestrs, $typetuple, $nexpr)
  97. end
  98. end
  99. function do_quickcheck(f :: Function, exprstr, varnames, types :: NTuple{N,DataType}, n :: Integer) where {N}
  100. rng = MersenneTwister(0)
  101. for vars in specialcases(types)
  102. do_quickcheck(f, exprstr, varnames, vars) || return false
  103. end
  104. for _ in 1:n
  105. vars = generate(rng, types)
  106. do_quickcheck(f, exprstr, varnames, vars) || return false
  107. end
  108. return true
  109. end
  110. function do_quickcheck(f :: Function, exprstr, varnames, vars)
  111. try
  112. if !f(vars...)
  113. if length(varnames) == 1
  114. x = Expr(:(=), Symbol(varnames[1]), vars[1])
  115. else
  116. x = Expr(:tuple, (Expr(:(=), n, v) for (n, v) in zip(map(Symbol, varnames), vars))...)
  117. end
  118. @warn "Property `$exprstr` does not hold for $x."
  119. return false
  120. end
  121. catch exception
  122. if length(varnames) == 1
  123. x = Expr(:(=), Symbol(varnames[1]), vars[1])
  124. else
  125. x = Expr(:tuple, (Expr(:(=), n, v) for (n, v) in zip(map(Symbol, varnames), vars))...)
  126. end
  127. @warn "Property `$exprstr` does not hold for $x."
  128. rethrow(exception)
  129. end
  130. return true
  131. end
  132. """
  133. generate(rng :: Random.AbstractRNG, T :: DataType)
  134. Returns a single pseudorandomly chosen specimen corresponding to data type `T` for the `@quickcheck` macro.
  135. `RandomPropertyTest` defines this function for some builtin types, for example `Int` and `Float32`.
  136. To define a generator for your own custom type, `import RandomizedPropertyTest.generate` and specialize it for that type.
  137. See also `specialcases`, `@quickcheck`.
  138. Example
  139. -------
  140. Specialize `generate` to generate double-precision floats in the interval [0, π):
  141. ```jldoctest
  142. julia> import Random
  143. julia> struct MyRange; end
  144. julia> RandomizedPropertyTest.generate(rng :: Random.AbstractRNG, _ :: Type{MyRange}) = rand(rng, Float64) * π;
  145. ```
  146. This is just an example; in practice, consider using `Range{Float64, 0, π}` instead.
  147. """
  148. function generate(rng :: AbstractRNG, types :: NTuple{N, DataType}) where {N}
  149. return (generate(rng, T) for T in types)
  150. end
  151. # Special cases for small numbers of variables increases performance by (15%, 30%, 40%, 40%) for (one, two, three, four) variables, respectively.
  152. @inline function generate(rng :: AbstractRNG, types :: NTuple{1, DataType})
  153. return (generate(rng, types[1]),)
  154. end
  155. @inline function generate(rng :: AbstractRNG, types :: NTuple{2, DataType})
  156. return (generate(rng, types[1]), generate(rng, types[2]))
  157. end
  158. @inline function generate(rng :: AbstractRNG, types :: NTuple{3, DataType})
  159. return (generate(rng, types[1]), generate(rng, types[2]), generate(rng, types[3]))
  160. end
  161. @inline function generate(rng :: AbstractRNG, types :: NTuple{4, DataType})
  162. return (generate(rng, types[1]), generate(rng, types[2]), generate(rng, types[3]), generate(rng, types[4]))
  163. end
  164. function generate(rng :: AbstractRNG, _ :: Type{T}) where {T}
  165. rand(rng, T)
  166. end
  167. function generate(rng :: AbstractRNG, _ :: Type{Array{T,N}}) :: Array{T,N} where {T,N}
  168. shape = Int.(round.(1 .+ 3 .* randexp(rng, N))) # empty array is a special case
  169. return rand(rng, T, shape...)
  170. end
  171. for (TI, TF) in Dict(Int16 => Float16, Int32 => Float32, Int64 => Float64)
  172. @eval begin
  173. function generate(rng :: AbstractRNG, _ :: Type{$TF})
  174. x = $TF(NaN)
  175. while !isfinite(x)
  176. x = reinterpret($TF, rand(rng, $TI)) # generate a random int and pretend it is a float.
  177. # This gives an extremely broad distribution of floats.
  178. # Around 1% of the floats will have an absolute value between 1e-3 and 1e3.
  179. end
  180. return x
  181. end
  182. end
  183. end
  184. function generate(rng :: AbstractRNG, _ :: Type{Complex{T}}) where {T<:AbstractFloat}
  185. return complex(generate(rng, T), generate(rng, T))
  186. end
  187. """
  188. specialcases(T :: DataType)
  189. Returns a one-dimensional Array of values corresponding to `T` for the `@quickcheck` macro.
  190. `RandomPropertyTest` overloads this function for some builtin types, for example `Int` and `Float64`.
  191. To define special cases for your own custom data type, `import RandomizedPropertyTest.specialcases` and specialize it for that type.
  192. See also `generate()`, `@quickcheck`.
  193. Examples
  194. --------
  195. View special cases for the builtin type `Bool`:
  196. ```jldoctest
  197. julia> RandomizedPropertyTest.specialcases(Bool)
  198. 2-element Array{Bool,1}:
  199. true
  200. false
  201. ```
  202. Define special cases for a custom type:
  203. ```
  204. julia> struct MyFloat; end
  205. julia> RandomizedPropertyTest.specialcases(_ :: Type{MyFloat}) = Float32[0.0, Inf, -Inf, eps(0.5), π];
  206. ```
  207. """
  208. function specialcases()
  209. return []
  210. end
  211. function specialcases(types :: NTuple{N,DataType}) where {N}
  212. return Base.product((specialcases(T) for T in types)...)
  213. end
  214. function specialcases(_ :: Type{Array{T,N}}) :: Array{Array{T,N},1} where {T,N}
  215. d0 = [Array{T,N}(undef, repeat([0], N)...)]
  216. d1 = [reshape([x], repeat([1], N)...) for x in specialcases(T)]
  217. if N 3
  218. # For N 3, this uses huge amounts of memory, so we don't do it.
  219. d2_ = collect(Base.Iterators.product(repeat([specialcases(T)], 2^N)...))
  220. d2 = [Array{T,N}(reshape([d2_[i]...], repeat([2], N)...)) for i in 1:length(d2_)]
  221. else
  222. d2 = Array{Array{T,N},1}(undef, 0)
  223. end
  224. return cat(d0, d1, d2, dims=1)
  225. end
  226. function specialcases(_ :: Type{T}) :: Array{T,1} where {T<:AbstractFloat}
  227. return [
  228. T(0.0),
  229. T(1.0),
  230. T(-1.0),
  231. T(-0.5),
  232. T(1)/T(3),
  233. floatmax(T),
  234. floatmin(T),
  235. maxintfloat(T),
  236. -one(T) / maxintfloat(T),
  237. T(NaN),
  238. T(Inf),
  239. T(-Inf),
  240. ]
  241. end
  242. function specialcases(_ :: Type{Complex{T}}) :: Array{Complex{T},1} where {T <: AbstractFloat}
  243. return [complex(r, i) for r in specialcases(T) for i in specialcases(T)]
  244. end
  245. function specialcases(_ :: Type{T}) :: Array{T,1} where {T <: Signed}
  246. smin = one(T) << (8 * sizeof(T) - 1)
  247. smax = smin - one(T)
  248. return [
  249. T(0),
  250. T(1),
  251. T(-1),
  252. T(2),
  253. T(-2),
  254. smax,
  255. smin
  256. ]
  257. end
  258. function specialcases(_ :: Type{T}) :: Array{T} where {T <: Integer}
  259. return [
  260. T(0),
  261. T(1),
  262. T(2),
  263. ~T(0),
  264. ]
  265. end
  266. function specialcases(_ :: Type{Bool}) :: Array{Bool,1}
  267. return [
  268. true,
  269. false,
  270. ]
  271. end
  272. #=
  273. special datatypes
  274. =#
  275. """
  276. Range{T,a,b}
  277. Represents a range of variables of type `T`, with both endpoints `a` and `b` included.
  278. `a` should be smaller than or eqaul to `b`.
  279. Both `a` and `b` should be finite and non-NaN.
  280. The type is used to generate variables of type `T` in the interval [`a`, `b`] for the `@quickcheck` macro:
  281. ```
  282. julia> @quickcheck (typeof(x) == Int && 23 ≤ x ≤ 42) (x :: Range{Int, 23, 42})
  283. true
  284. ```
  285. """
  286. # Note: Having a and b (the range endpoints) as type parameters is a bit unfortunate.
  287. # It means that for ranges with the same type T but different endpoints, all relevant functions have to be recompiled.
  288. # However, it is required because of generate(::NTuple{N, DataType}).
  289. # Anyway, it is only for tests, so it should not be too much of a problem.
  290. struct Range{T,a,b} end
  291. export Range
  292. function generate(rng :: AbstractRNG, _ :: Type{Range{T,a,b}}) :: T where {T<:AbstractFloat,a,b}
  293. a b && isfinite(a) && isfinite(b) || error("a needs to be ≤ b and both need to be finite")
  294. a + rand(rng, T) * (b - a) # The endpoints are included via specialcases()
  295. end
  296. function generate(rng :: AbstractRNG, _ :: Type{Range{T,a,b}}) :: T where {T<:Integer,a,b}
  297. a b || error("a needs to be ≤ b")
  298. rand(rng, a:b)
  299. end
  300. function specialcases(_ :: Type{Range{T,a,b}}) :: Array{T,1} where {T<:AbstractFloat,a,b}
  301. return [
  302. T(a),
  303. T(a) + (T(b)/2 - T(a)/2),
  304. T(b),
  305. ]
  306. end
  307. function specialcases(_ :: Type{Range{T,a,b}}) :: Array{T,1} where {T<:Integer,a,b}
  308. return [
  309. T(a),
  310. div(T(a)+T(b), 2),
  311. T(b),
  312. ]
  313. end
  314. """
  315. Disk{T,z,r}
  316. Represents a Disk of radius `r and center `z` in the set `T` (boundary excluded).
  317. `r` should be nonnegative.
  318. Both `z` and `r` should be finite and non-NaN.
  319. The type is used to generate variables `x` of type `T` such that (abs(x-z) < r) for the `@quickcheck` macro:
  320. ```
  321. julia> @quickcheck (typeof(x) == ComplexF16 && abs(x-2im) < 3) (x :: Disk{Complex{Float16}, 2im, 3})
  322. true
  323. ```
  324. """
  325. struct Disk{T,z₀,r} end
  326. export Disk
  327. function generate(rng :: AbstractRNG, _ :: Type{Disk{Complex{T},z₀,r}}) :: Complex{T} where {T<:AbstractFloat,z₀,r}
  328. r ≥ 0 || error("r needs to be ≥ 0")
  329. isfinite(z₀) || error("z₀ needs to be finite")
  330. z = Complex{T}(Inf, Inf)
  331. while !(abs(z - z₀) < r)
  332. z = r * complex(2rand(rng, T)-1, 2rand(rng, T)-1) + z₀
  333. end
  334. return z
  335. end
  336. function specialcases(_ :: Type{Disk{Complex{T},z₀,r}}) :: Array{Complex{T},1} where {T<:AbstractFloat,z₀,r}
  337. return [
  338. Complex{T}(z₀)
  339. ]
  340. end
  341. end