Adjustable tolerance for nullspace (#26032)

* adjustable tolerance for nullspace

* trailing whitespace

* thin SVD is actually fine, we don't access U

* use smallest dimension for default tolerance

* typo, and we do need full=true

* fix test

stdlib/LinearAlgebra/src/dense.jl

@@ -1310,9 +1310,13 @@ end
 ## Basis for null space
 
 """
-    nullspace(M)
+    nullspace(M[, tol::Real])
 
-Basis for nullspace of `M`.
+Computes a basis for the nullspace of `M` by including the singular
+vectors of A whose singular have magnitude are greater than `tol*σ₁`,
+where `σ₁` is `A`'s largest singular values. By default, the value of
+`tol` is the smallest dimension of `A` multiplied by the [`eps`](@ref)
+of the [`eltype`](@ref) of `A`.
 
 # Examples
 ```jldoctest
@@ -1327,16 +1331,22 @@ julia> nullspace(M)
  0.0
  0.0
  1.0
+
+julia> nullspace(M, 2)
+3×3 Array{Float64,2}:
+ 0.0  1.0  0.0
+ 1.0  0.0  0.0
+ 0.0  0.0  1.0
 ```
 """
-function nullspace(A::StridedMatrix{T}) where T
+function nullspace(A::StridedMatrix, tol::Real = min(size(A)...)*eps(real(float(one(eltype(A))))))
     m, n = size(A)
     (m == 0 || n == 0) && return Matrix{T}(I, n, n)
-    SVD = svdfact(A, full = true)
-    indstart = sum(SVD.S .> max(m,n)*maximum(SVD.S)*eps(eltype(SVD.S))) + 1
+    SVD = svdfact(A, full=true)
+    indstart = sum(SVD.S .> SVD.S[1]*tol) + 1
     return copy(SVD.Vt[indstart:end,:]')
 end
-nullspace(a::StridedVector) = nullspace(reshape(a, length(a), 1))
+nullspace(a::StridedVector, tol::Real = min(size(a)...)*eps(real(float(one(eltype(a)))))) = nullspace(reshape(a, length(a), 1), tol)
 
 """
     cond(M, p::Real=2)

stdlib/LinearAlgebra/test/dense.jl

@@ -72,7 +72,9 @@ bimg  = randn(n,2)/2
                 @test norm(a[:,1:n1]'a15null,Inf) ≈ zero(eltya) atol=300ε
                 @test norm(a15null'a[:,1:n1],Inf) ≈ zero(eltya) atol=400ε
                 @test size(nullspace(b), 2) == 0
+                @test size(nullspace(b, 100*εb), 2) == 0
                 @test nullspace(zeros(eltya,n)) == Matrix(I, 1, 1)
+                @test nullspace(zeros(eltya,n), 0.1) == Matrix(I, 1, 1)
             end
         end
     end # for eltyb