"""
    closest_pair_search(array::Array{Int64,1}, target::Int64)

Finding the closet pair of values for a given target in a sorted array. For this reason,
the distance between a pair of of values has to be minimazied with respect to the target.
For more information see: [https://en.wikipedia.org/wiki/Closest_pair_of_points_problem](https://en.wikipedia.org/wiki/Closest_pair_of_points_problem)

# Arguments
- `array::Array{Int64,1}`: Sorted array of integers
- `target::Int64`: Target-value to find the position

# Examples
```julia-repl
julia> import ClassicAlgorithmsCollections
julia> arr = [10, 22, 28, 29, 30, 40]
julia> target = 56
julia> ClassicAlgorithmsCollections.closest_pair_search(arr, target)
(28, 29)
```
"""
function closest_pair_search(array::Array{Int64,1}, target::Int64)

    # To store indexes of result pair
    res_left, res_right = 1, 1

    #Initialize left, right array boundaries, and their difference
    left, right, diff = 1, length(array), 999999

    # While there are elements between left and right array boundaries
    while right > left

        # Check if the distance between the current pair is closer than the closest
        # previous pair so far
        if abs(array[left] + array[right] - target) < diff
            res_left = left
            res_right = right
            # If yes update the distance
            diff = abs(array[left] + array[right] - target)
        end

        # If this pair has more sum than the target, shrink the array to a smaller value
        if array[left] + array[right] > target

            right -= 1
        # Otherwise, shrink the array to a larger value
        else
            left += 1
        end
    end
    return array[res_left], array[res_right]
end


function euler(f::Function, x::Float64, y::Vector{Float64}, steps::Int64, h::Float64)
  nsteps = steps + 1
  res = zeros(nsteps, length(y)+1)
  for i in 1:nsteps
    res[i,:] = [x; [y[i] for i in 1:length(y)]]
    k0 = h * f(x, y)
    y += k0
    x += h
  end
  res
end

function modified_euler(f::Function, x::Float64, y::Vector{Float64}, steps::Int64, h::Float64)
  nsteps = steps + 1
  res = zeros(nsteps, length(y)+1)
  for i in 1:nsteps
    res[i,:] = [x; [y[i] for i in 1:length(y)]]
    k0 = h * f(x, y)
    k1 = h * f(x+h, y+k0)
    y += (k0+k1)/2.0
    x += h
  end
  res
end

function mid_point_euler(f::Function, x::Float64, y::Vector{Float64}, steps::Int64, h::Float64)
  nsteps = steps + 1
  res = zeros(nsteps, length(y)+1)
  for i in 1:nsteps
    res[i,:] = [x; [y[i] for i in 1:length(y)]]
    k0 = h * f(x, y)
    k1 = h * f(x+h/2.0, y+k0/2.0)
    y += k1
    x += h
  end
  res
end

function runga_kutta_4(f::Function, x::Float64, y::Vector{Float64}, steps::Int64, h::Float64)
  nsteps = steps + 1
  res = zeros(nsteps, length(y)+1)
  for i in 1:nsteps
    res[i,:] = [x; [y[i] for i in 1:length(y)]]
    k0 = h * f(x, y)
    k1 = h * f(x+h/2.0, y+k0/2.0)
    k2 = h * f(x+h/2.0, y+k1/2.0)
    k3 = h * f(x+h/2.0, y+k2)
    y += (k0+2.0*k1+2.0*k2+k3)/6.0
    x += h
  end
  res
end


# R2 attributed

using Distributions

n = 10000
x1 = rand(Normal(), n)
x2 = rand(Normal(), n) .+ .2 * x1
x3 = 1000*(rand(Normal(), n) .- .2 * x1 .+ .2 * x2)

x2 = rand(Normal(), n)
x3 = rand(Normal(), n)

u = rand(Normal(), n) .* 2

y = 2 .+ 1.5*x1 .+ 2.7*x2 .- .5* x3 + u

constant = fill(1,n)
X = hcat(constant, x1, x2, x3)

β = inv(X'X) * X'y

ŷ = X*β

s² = ((y-ŷ)' * (y-ŷ)) / (n-4)

seβ₁ = √(s² * inv(x1' * x1))
seβ₂ = √(s² * inv(x2' * x2))
seβ₃ = √(s² * inv(x3' * x3))

√(s² .* inv(X'X))

ccdf(Chisq(4), s²)

SS(x) = sum((x .- mean(x)).^2)

# Sum of Squares Total
SST = SS(y)

# Sum of Squares Explained
SSR = SS(ŷ)

# Sum of Squares Error
SSE = SS(y-ŷ)

# Degrees of freedom model
DFM = 4-1

# Degrees of freedom total
DFT = n-4

# Mean of squares of regression
MSR = SSR/DFM

# Mean of squares of total
MST = SSR/DFT

F = MSR/MST

R2 = SSR/SST

ŷ₁ = x1*β[2]
ŷ₂ = x2*β[3]
ŷ₃ = x3*β[4]

SSR1 = SS(ŷ₁)
SSR2 = SS(ŷ₂)
SSR3 = SS(ŷ₃)

SSR
SSRp = SSR1 + SSR2 + SSR3

SSR1p = SSR1/SSRp
SSR2p = SSR2/SSRp
SSR3p = SSR3/SSRp

##### Normalization Approach
x1 = x1/var(x1)^.5
x2 = x2/var(x2)^.5
x3 = x3/var(x3)^.5

X = hcat(constant, x1, x2, x3)

β = inv(X'X) * X'y

# Asserting Independence - generally vioated
# var(y) = β[2]^2*var(x1) + β[3]^2*var(x2) + β[4]^2*var(x3)

# Since we normalized
denom = β[2]^2 + β[3]^2 + β[4]^2

β[2]^2/denom
β[3]^2/denom
β[4]^2/denom


function func(x)
    return x-cos(x)
end

function bisection(func,xleft,xright,eps)
    y_left = func(xleft)
    y_right = func(xright)
    num_iter = 0
    iteration = 100000000
    if y_left == 0
        return xleft, num_iter
    elseif y_right == 0
        return xright, num_iter
    elseif y_left*y_right > 0
        return nothing, nothing
    else
        for n in 1:iteration
            x_mid = (xleft+xright)/2
            y_mid = func(x_mid)
            if y_mid == 0
                num_iter = n
                return x_mid, num_iter
            end
            if y_left*y_mid < 0
                xright = x_mid
            else
                xleft = x_mid
            end
            if(abs(xright-xleft) < eps)
                num_iter = n
                return x_mid, num_iter
            end
        end
        return nothing, num_iter
    end
end

function main()
    xleft, xright = 0, 3.14
    eps = 1e-7
    solution, num_iter = bisection(func, xleft, xright, eps)
    if solution != nothing
        print("X=$(solution) $(func(solution)), $(num_iter)")
    elseif num_iter == nothing
        print("f(x1)*f(x2) must be less than 0")
    else
        print("No Convergence")
    end
end

main()


"""
perfect_square(number)

Check if a number is a perfect square or not.

# Example
```jula
perfect_square(9)   # returns True
perfect_square(16)  # returns True
perfect_square(1)   # returns True
perfect_square(0)   # returns True
perfect_square(10)  # returns False
perfect_square(-9)  # returns False
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee) and [Rratic](https://github.com/Rratic)
"""
function perfect_square(number::N)where N<:Integer
    number<0&&return false
    val=isqrt(number)
    return val*val == number
end


include("factorial.jl")

"""
    krishnamurthy(number)

Check if a number is a Krishnamurthy number or not

# Details

It is also known as Peterson Number.

A Krishnamurthy Number is a number whose sum of the
factorial of the digits equals to the original
number itself.

For example: 145 = 1! + 4! + 5!
    So, 145 is a Krishnamurthy Number

# Example

```julia
krishnamurthy(145) # returns true
krishnamurthy(240) # returns false
krishnamurthy(1)   # returns true
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function krishnamurthy(number)
    if number != trunc(number) || number < 0
        throw(
            error("krishnamurthy() only accepts non-negative integral values"),
        )
    end
    total = 0
    temp = number
    while temp > 0
        temp, digit = divrem(temp, 10)
        total += factorial_iterative(digit)
    end
    return total == number
end


"""
    monte_carlo_integration(f::Function, a::Real, b::Real, n::Int)

Monte carlo integration is a very easy and scalable way to do multidimentional integrals.
However, only single variable integrals are considered.

# Arguments
- `f`: the function to integrate. (at the momment only single variable is suported)
- `a`: start in the integration limits.
- `b`: endin the integration limits.
- `N`: Number of points to sample. For most simple functions, 1000 to 10,000 should be okay.

# Examples
```julia
julia> monte_carlo_integration(x -> 3*x^2, 0, 1, 100000) # integrate a polynomial
1.0000037602209

julia> monte_carlo_integration(x -> sin(x), 0, pi, 1000) # integrate the sin function
2.0018927826323756
```

# References
- https://www.scratchapixel.com/lessons/mathematics-physics-for-computer-graphics/monte-carlo-methods-in-practice/monte-carlo-integration
- https://kingaa.github.io/sbied/pfilter/monteCarlo.html

# Contributors
- [AugustoCL](https://github.com/AugustoCL)
- [Ved Mahajan](https://github.com/Ved-Mahajan)
"""
function monte_carlo_integration(f::Function, a::Real, b::Real, n::Int)
    Δₓ = ((b - a) / n)

    Σ = 0.0
    for _ ∈ 1:n
        # generate uniform(a, b) using uniform(0, 1)
        Xᵢ = a + (b-a)*rand()
        Σ += f(Xᵢ)
    end

    return Δₓ * Σ
end


function solve_poisson_cg( LF::LF3dGrid, rho::Array{Float64,1}, NiterMax::Int64;
                           verbose=false, TOL=5.e-10 )
    Npoints = LF.Nx * LF.Ny * LF.Nz
    #
    phi = zeros( Float64, Npoints ) # XXX or use some starting guess
    #
    r = zeros( Float64, Npoints )
    p = zeros( Float64, Npoints )
    #
    nabla2_phi = apply_Laplacian( LF, phi )
    for ip = 1:Npoints
        r[ip] = rho[ip] - nabla2_phi[ip]
        p[ip] = r[ip]
    end

    rsold = dot( r, r )
    for iter = 1 : NiterMax
        #
        nabla2_phi = apply_Laplacian( LF, p )
        #
        alpha = rsold/dot( p, nabla2_phi )
        #
        phi = phi + alpha * p
        r = r - alpha * nabla2_phi
        #
        rsnew = dot( r, r )
        deltars = rsold - rsnew # used ?
        if verbose
          @printf("%8d %20.10f\n", iter, sqrt(rsnew))
        end
        #
        if sqrt(rsnew) < TOL
          if verbose
            @printf("#Convergence achieved in solve_poison_cg: N, iter: %d %d\n", Npoints, iter)
          end
          break
        end
        #
        p = r + (rsnew/rsold) * p
        #
        rsold = rsnew
    end
    #
    return phi
    #
end # of function

function solve_poisson_cg( Lmat::SparseMatrixCSC{Float64,Int64},
                           rho::Array{Float64,1}, NiterMax::Int64;
                           verbose=false, TOL=5.e-10 )
    #
    Npoints = size(rho)[1]
    phi = zeros( Float64, Npoints ) # XXX or use some starting guess
    #
    r = zeros( Float64, Npoints )
    p = zeros( Float64, Npoints )
    #
    nabla2_phi = Lmat*phi
    for ip = 1 : Npoints
        r[ip] = rho[ip] - nabla2_phi[ip]
        p[ip] = r[ip]
    end
    rsold = dot( r, r )

    for iter = 1 : NiterMax
        #
        nabla2_phi = Lmat*p
        #
        alpha = rsold/dot( p, nabla2_phi )
        #
        phi = phi + alpha * p
        r = r - alpha * nabla2_phi
        #
        rsnew = dot( r, r )
        deltars = rsold - rsnew
        if verbose
            @printf("%8d %20.10f\n", iter, sqrt(rsnew))
        end
        #
        if sqrt(rsnew) < TOL
            if verbose
                @printf("Convergence achieved in solve_poison_cg: N, iter: %d %d\n", Npoints, iter)
            end
            break
        end
        #
        p = r + (rsnew/rsold) * p
        #
        rsold = rsnew
    end
    #
    return phi
    #
end # of function


"""
    sum_ap(first_term, diff, num_terms)

Finds sum of a arithmetic progression series

# Input parameters

- first_term : first term of the series
- diff       : common difference between consecutive terms
- num_terms  : number of terms in the series till which we count sum

# Example

```julia
sum_ap(1, 1, 10)    # returns 55.0
sum_ap(1, 10, 100)  # returns 49600.0
```

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function sum_ap(first_term, diff, num_terms)
    #formula for sum of the ap series
    sum = (num_terms / 2) * (2 * first_term + (num_terms - 1) * diff)
    return sum
end


"""
    Pearson Correlation

Find the pearson correlation between two variables.

# Example:

julia> PearsonCorrelation([12,11,16,17,19,21],[11,51,66,72,12,15])
-0.2092706263573845

Contribution by: [Aru Bhardwaj](https://github.com/arubhardwaj)

"""

function PearsonCorrelation(x, y)
    mean_x = sum(x)/length(x)
    mean_y = sum(y)/length(y)
    XY = (x.-mean_x).*(y.-mean_y)
    XXs = sum((x.-mean_x).*(x.-mean_x))
    YYs = sum((y.-mean_y).*(y.-mean_y))
    return(sum(XY)/(sqrt(XXs.*YYs)))
end


eval_at(args...) = f -> f(args...)
findallin(xs, ys) = map(x -> findfirst(y -> x == y, ys), xs)

"""
    catalan_number(n)

calculate the n-th catalan number
"""
catalan_number(n) = div(binomial(2n, n), n+1)

"""
    number_binary_trees(n)

calculate the number of binary trees with n leafs
"""
number_binary_trees(n) = catalan_number(n - 1)

"""
    categorical_sample(tokens, weights)

sample one token according to the given weights
"""
function categorical_sample(tokens, weights)
    T = eltype(weights)
    x = convert(T, rand()) * sum(weights)
    cum_weights = zero(T)
    for (t, w) in zip(tokens, weights)
        cum_weights += w
        if cum_weights >= x
            return t
        end
    end
    error("categorical sample failed with length(tokens)=$(length(tokens)), weights=$weights")
end

categorical_sample(d::Dict)   = categorical_sample(keys(d), values(d))
categorical_sample(v::Vector) = categorical_sample(1:length(v), v)


using Plots

"""
    runge_kutta(f::Function, x₀::Real, y₀::Real, x::Real, h::Real=10^(-3))::Array{Float64,1}
Return an array with values of function which is solution to the differential equation
given by function f with intial condition
"""
function runge_kutta(f::Function, x₀::Real, y₀::Real, x::Real, h::Real=10^(-3))::Array{Float64,1}
    n = round(Int64, x / h)
    array = zeros(n)
    for i in 1:n
        k₁ = f(x₀, y₀)
        k₂ = f(x₀ + 1/2*h, y₀ + 1/2*h*k₁)
        k₃ = f(x₀ + 1/2*h, y₀ + 1/2*h*k₂)
        k₄ = f(x₀ + h, y₀ + h*k₃)
        y₀ = y₀ + h / 6 * (k₁ + 2*k₂ + 2*k₃ + k₄)
        x₀ = x₀ + h
        array[i] = y₀
    end
    return array
end

# logistic growth for chosen constants
f(t, y) = 0.6y*(1 - y / 10)
array = runge_kutta(f, 0, 1, 20)
𝐫 = plot(LinRange(0, 20, length(array)), array)
display(𝐫)

# https://en.smath.com/wiki/GetFile.aspx?File=Examples/RK4SystemEquations.pdf
# examples below are taken from this url

array  = [(t, y₁, y₂) -> sin(t) + cos(y₁) + sin(y₂),
        (t, y₁, y₂) -> cos(t) + sin(y₂)]
X = 0
Y = [-1., 1.]

"""
    evaluate(array::Array{Function, 1}, X::Real, Y::Array{Real, 1})::Array{Real, 1}
Return an array with values of function f(t, y₁, y₂) defined in array for given point X
"""
function evaluate(array::Array{Function, 1}, X::Real, Y::Array{T, 1})::Array{T, 1} where T <: Real
    return [i(X, Y[1], Y[2]) for i in array]
end

"""
    runge_kutta(F::Array{Function, 1}, x::Real, X::Real, Y::Array{T, 1}, h::Real=1e-3)::Array{T, 2} where T <: Real
Return an array with values of functions which are solutions to the system of differential equations defined by the array.
X and Y are initial conditions, x is the maximum point.
"""
function runge_kutta(F::Array{Function, 1}, x::Real, X::Real, Y::Array{T, 1}, h::Real=1e-3)::Array{T, 2} where T <: Real
    n = round(Int64, x / h)
    array = zeros((n, 2))
    for i in 1:n
        k₁ = evaluate(F, X, Y)
        k₂ = evaluate(F, X + 1/2 * h, Y .+ 1/2 * h * k₁)
        k₃ = evaluate(F, X + 1/2 * h, Y .+ 1/2 * h * k₂)
        k₄ = evaluate(F, X + h, Y .+ h * k₃)
        Y = @. Y + h / 6 * (k₁ + 2*k₂ + 2*k₃ + k₄)
        X = X + h
        array[i, :] = Y
    end
    return array
end

a = runge_kutta(array, 20, X, Y)

𝑝 = plot(LinRange(0, 5, length(a[:, 1])), a[:, 1])
plot!(LinRange(0, 5, length(a[:, 2])), a[:, 2])
display(𝑝)
# everything works fine

# harmonic oscillator ẍ + x = 0
# can be rewritten as
# ẋ = y, ẏ = -x
# note that y is velocity ⃗v

array2 = [(t, y₁, y₂) -> y₂, (t, y₁, y₂) -> -y₁]
Y2 = [0., 1.]
b = runge_kutta(array2, 30, 0, Y2)

𝑞 = plot(LinRange(0, 5, length(b[:, 1])), b[:, 1])
display(𝑞)
# plot!(LinRange(0, 5, length(b[:, 2])), b[:, 2])
𝑤 = plot(b[:, 1], b[:, 2])  # a kind of phase plane plot
display(𝑤)

# vector field for harmonic oscillator
xs = LinRange(0, 5, length(b[1, :]))

xs = xs[1:150:end]
xs |>println
y₁s = b[:, 1][1:150:end]
y₂s = b[:, 2][1:150:end]

plot(quiver(y₁s, y₂s, quiver=(0.2 * array2[1].(xs, y₁s, y₂s), 0.2 * array2[2].(xs, y₁s, y₂s)), color=:red))


function bin_length_long(s::AbstractString)
    binNum = parse(UInt, s)

    finNum = 0
    seq = 1

    for i in 1:binNum
        if (i == seq)
            finNum += 1
            seq *= 2
        end
    end

    return string(finNum)
end

"""
This algorithm features use of the OEIS entry A070939 -
length of Binary Representation of n. It finds
the length of any binary number and returns said length.

 https://oeis.org/A070939

This function, as believed, is O(n)

The idea follows that the sequence is dependent on
a repeating pattern of 2. Short for sequence, seq is the
number of digits required before an increase in finNum
or final number as seen here with the first few
iterations - i on the left, final number on the right:

1 : 1
2 : 2
3 : 2
4 : 3
5 : 3
6 : 3
7 : 3
8 : 4
cont.

As you can see, for every version of i, there is an appropriate
modified number that only repeats for the sequence of the last
doubled amount.

#Contributions:
Contributed by F35H: https://github.com/F35H
"""

function bin_length_short(s::AbstractString)
    binNum = parse(UInt, s)

    finNum = 0
    i = 1

    while i <= binNum
        i *= 2
        finNum += 1
    end

    return string(finNum)
end

"""
This algorithm features use of the OEIS entry A070939 -
length of Binary Representation of n. It finds
the length of any binary number and returns said length.

 https://oeis.org/A070939

This function, as believed, is O(n)

The idea follows that the sequence is dependent on
a repeating pattern of 2. The final number being finNum
increases on every doubling of i.

1 : 1
2 : 2
3 : 2
4 : 3
5 : 3
6 : 3
7 : 3
8 : 4
cont.

As you can see, for every version of i, there is an appropriate
final number that iterates on every doubling of i.

Contributors:
- [F45H](https://github.com/F35H)
"""


"""
    Newton_Raphson(f, fp, x; tol = 1e-14, MITR = 100)
---
Given function `f` and its derivative `fp`, this function finds a root for `f = 0` with
a starter `x`
"""
function Newton_Raphson(f, fp, x; tol = 1e-14, MITR = 100)
    xnew, xold = x, Inf
    fn, fo = f(xnew), Inf

    ctr = 1

    while (ctr < 100) && (abs(xnew - xold) > tol) && ( abs(fn - fo) > tol )
	x = xnew - f(xnew)/fp(xnew) # update step
	xnew, xold = x, xnew
        fn, fo = f(xnew), fn
	ctr = ctr + 1
    end

    if ctr == MITR
	error("Did not converge in $MITR steps")
    else
	xnew, ctr
    end
end

"""
    Newton_Raphson(f, x; tol = 1e-14, MITR = 100, inc = 1e-8)
---
Given function `f`, this function numerically finds a root for `f = 0` with a starter `x` using `secant` method.
"""
function Newton_Raphson(f, x; tol = 1e-14, MITR = 100, inc = 1e-8)
    fp(x1, x2) = (f(x1) - f(x2))/(x1 - x2)

    fn, fo = f(x1), Inf

    ctr = 1

    x2, x1 = x + inc, x
    while (ctr < 100) && (abs(x2 - x1) > tol) && ( abs(fn - fo) > tol )
        x2, x1 = x2 - f(x2)/fp(x1, x2), x2
        fn, fo = f(x2), fn
	ctr = ctr + 1
    end

    if ctr == 100
	error("Did not converge in 100 steps")
    else
	x2, ctr
    end
end


using SymPy
using Base.Test

## ODEs
x, a = Sym("x, a")
F = SymFunction("F")
ex = diff(F(x), x) - a*F(x)
ex1 = dsolve(ex)
ex2 = rhs(ex1) |> subs(Sym(:C1), 1) |> subs(a, 2)
@assert ex2 == exp(2x)

t, = @syms t
X, Y = map(SymFunction, ("X", "Y"))
eq = [Eq(diff(X(t),t), 12*t*X(t) + 8*Y(t)), Eq(diff(Y(t),t), 21*X(t) + 7*t*Y(t))]
dsolve(eq)

## version 0.4+ allow use of u'(x) in lieu of diff(u(x), x) and `ivpsolve`
u = SymFunction("u")
a,x, y, y0, y1 = symbols("a, x, y, y0, y1")

dsolve(u'(x) - a*u(x), x, (u, 0, 1))
dsolve(u'(x) - a*u(x), x, (u, 0, y1))
dsolve(u'(x) - a*u(x), x, (u, y0, y1))
dsolve(x*u'(x) + x*u(x) + 1, x, (u, 1, 1))
dsolve((u'(x))^2 - a*u(x), x, (u, 0, 1))
dsolve(u''(x) - a * u(x), x, (u, 0, 1), (u', 0, 0))

F, G, K = symbols("F, G, K", cls = symfunction)
eqn = F(x)*u'(y)*y + G(x)*u(y) + K(x)
dsolve(eqn, y, (u, 1, 0))

## dsolve eqn has two answers, but we want to eliminate based on initial condition
dsolve(u'(x) - (u(x)-1)*u(x)*(u(x)+1), x, (u, 0, 1//2))


@doc raw"""
    block_matrix_logdet(A, B, C, D)

Log-determinant of a block matrix using the determinant lemma.

```math
\det\left(
    \begin{bmatrix}
        \mathbf{A} & \mathbf{B} \\
        \mathbf{C} & \mathbf{D}
    \end{bmatrix}
\right)
= \det(A) \det(D - CA^{-1}B)
= \det(D) \det(A - BD^{-1}C)
```

Here we assume that `A` and `D` are invertible, and moreover are easy to invert
(for example, if they are diagonal).
We use this to chose one or the other of the two formulas above.
"""
function block_matrix_logdet(
    A::AbstractMatrix, B::AbstractMatrix,
    C::AbstractMatrix, D::AbstractMatrix
)
    @assert size(A, 1) == size(B, 1)
    @assert size(C, 1) == size(D, 1)
    @assert size(A, 2) == size(C, 2)
    @assert size(B, 2) == size(D, 2)

    if length(A) ≥ length(D)
        return logdet(A) + logdet(D - C * inv(A) * B)
    else
        return logdet(D) + logdet(A - B * inv(D) * C)
    end
end

@doc raw"""
    block_matrix_invert(A, B, C, D)

Inversion of a block matrix, using the formula:

```math
\begin{bmatrix}
    \mathbf{A} & \mathbf{B} \\
    \mathbf{C} & \mathbf{D}
\end{bmatrix}^{-1}
=
\begin{bmatrix}
    \left(\mathbf{A} - \mathbf{B} \mathbf{D}^{-1} \mathbf{C}\right)^{-1} & \mathbf{0} \\
    \mathbf{0} & \left(\mathbf{D} - \mathbf{C} \mathbf{A}^{-1} \mathbf{B}\right)^{-1}
\end{bmatrix}
\begin{bmatrix}
    \mathbf{I} & -\mathbf{B} \mathbf{D}^{-1} \\
    -\mathbf{C} \mathbf{A}^{-1} & \mathbf{I}
\end{bmatrix}
```

Assumes that `A` and `D` are square and invertible.
"""
function block_matrix_invert(
    A::AbstractMatrix, B::AbstractMatrix,
    C::AbstractMatrix, D::AbstractMatrix
)
    @assert size(A, 1) == size(A, 2) == size(B, 1) == size(C, 2)
    @assert size(D, 1) == size(D, 2) == size(B, 2) == size(C, 1)

    CinvA = C * inv(A)
    BinvD = B * inv(D)

    M = [
        inv(A - BinvD * C)  zero(B)
        zero(C)  inv(D - CinvA * B)
    ]

    N = [
        I(size(B, 1))  -BinvD
        -CinvA  I(size(C, 1))
    ]

    return M * N
end


"""
euler(ivp,n)

Apply Euler's method to solve the given IVP using `n` time steps.
Returns a vector of times and a vector of solution values.
"""
function euler(ivp,n)
    # Time discretization.
    a,b = ivp.tspan
    h = (b-a)/n
    t = [ a + i*h for i in 0:n ]

    # Initial condition and output setup.
    u = fill(float(ivp.u0),n+1)

    # The time stepping iteration.
    for i in 1:n
        u[i+1] = u[i] + h*ivp.f(u[i],ivp.p,t[i])
    end
    return t,u
end

"""
ie2(ivp,n)

Apply the Improved Euler method to solve the given IVP using `n`
time steps. Returns a vector of times and a vector of solution
values.
"""
function ie2(ivp,n)
    # Time discretization.
    a,b = ivp.tspan
    h = (b-a)/n
    t = [ a + i*h for i in 0:n ]

    # Initialize output.
    u = fill(float(ivp.u0),n+1)

    # Time stepping.
    for i in 1:n
        uhalf = u[i] + h/2*ivp.f(u[i],ivp.p,t[i]);
        u[i+1] = u[i] + h*ivp.f(uhalf,ivp.p,t[i]+h/2);
    end
    return t,u
end

"""
rk4(ivp,n)

Apply "the" Runge-Kutta 4th order method to solve the given IVP
using `n` time steps. Returns a vector of times and a vector of
solution values.
"""
function rk4(ivp,n)
    # Time discretization.
    a,b = ivp.tspan
    h = (b-a)/n
    t = [ a + i*h for i in 0:n ]

    # Initialize output.
    u = fill(float(ivp.u0),n+1)

    # Time stepping.
    for i in 1:n
        k1 = h*ivp.f( u[i],      ivp.p, t[i]     )
        k2 = h*ivp.f( u[i]+k1/2, ivp.p, t[i]+h/2 )
        k3 = h*ivp.f( u[i]+k2/2, ivp.p, t[i]+h/2 )
        k4 = h*ivp.f( u[i]+k3,   ivp.p, t[i]+h   )
        u[i+1] = u[i] + (k1 + 2*(k2 + k3) + k4)/6
    end
    return t,u
end

"""
rk23(ivp,tol)

Apply an adaptive embedded RK formula pair to solve given IVP with
estimated error `tol`. Returns a vector of times and a vector of
solution values.
"""
function rk23(ivp,tol)
    # Initialize for the first time step.
    a,b = ivp.tspan
    t = [a]
    u = [float(ivp.u0)];   i = 1;
    h = 0.5*tol^(1/3)
    s1 = ivp.f(ivp.u0,ivp.p,a)

    # Time stepping.
    while t[i] < b
        # Detect underflow of the step size.
        if t[i]+h == t[i]
            @warn "Stepsize too small near t=$(t[i])"
            break  # quit time stepping loop
        end

        # New RK stages.
        s2 = ivp.f( u[i]+(h/2)*s1,   ivp.p, t[i]+h/2   )
        s3 = ivp.f( u[i]+(3*h/4)*s2, ivp.p, t[i]+3*h/4 )
        unew2 = u[i] + h*(2*s1 + 3*s2 + 4*s3)/9   # 2rd order solution
        s4 = ivp.f( unew2, ivp.p, t[i]+h )
        err = h*(-5*s1/72 + s2/12 + s3/9 - s4/8)  # 2nd/3rd difference
        E = norm(err,Inf)                         # error estimate
        maxerr = tol*(1 + norm(u[i],Inf))     # relative/absolute blend

        # Accept the proposed step?
        if E < maxerr     # yes
            push!(t,t[i]+h)
            push!(u,unew2)
            i += 1
            s1 = s4       # use FSAL property
        end

        # Adjust step size.
        q = 0.8*(maxerr/E)^(1/3)   # conservative optimal step factor
        q = min(q,4)               # limit stepsize growth
        h = min(q*h,b-t[i])        # don't step past the end
    end
    return t,u
end

"""
ab4(ivp,n)

Apply the Adams-Bashforth 4th order method to solve the given IVP
using `n` time steps.
"""
function ab4(ivp,n)
    # Time discretization.
    a,b = ivp.tspan
    h = (b-a)/n
    t = [ a + i*h for i in 0:n ]

    # Constants in the AB4 method.
    k = 4;    sigma = [55, -59, 37, -9]/24;

    # Find starting values by RK4.
    u = fill(float(ivp.u0),n+1)
    rkivp = ODEProblem(ivp.f,ivp.u0,(a,a+(k-1)*h),ivp.p)
    ts,us = rk4(rkivp,k-1)
    u[1:k] = us[1:k]

    # Compute history of u' values, from newest to oldest.
    f = [ ivp.f(u[k-i],ivp.p,t[k-i]) for i in 1:k-1  ]

    # Time stepping.
    for i in k:n
      f = [ ivp.f(u[i],ivp.p,t[i]), f[1:k-1]... ]   # new value of du/dt
      u[i+1] = u[i] + h*sum(f[j]*sigma[j] for j in 1:k)  # advance a step
    end
    return t,u
end

"""
am2(ivp,n)

Apply the Adams-Moulton 2nd order method to solve given IVP using
`n` time steps.
"""
function am2(ivp,n)
    # Time discretization.
    a,b = ivp.tspan
    h = (b-a)/n
    t = [ a + i*h for i in 0:n ]

     # Initialize output.
     u = fill(float(ivp.u0),n+1)

    # Time stepping.
    for i in 1:n
        # Data that does not depend on the new value.
        known = u[i] + h/2*ivp.f(u[i],ivp.p,t[i])
        # Find a root for the new value.
        F = z -> z .- h/2*ivp.f(z,ivp.p,t[i+1]) .- known
        unew = levenberg(F,known)
        u[i+1] = unew[end]
    end
    return t,u
end


using LinearAlgebra, Statistics;
function Newtonfixedpointmap(f, f_prime, x_0, tolerance, maxiter)
    # setup the algorithm
    x_old = x_0
    normdiff = Inf
    iter = 1
    while normdiff > tolerance && iter <= maxiter
        x_new = x_old - (f(x_old)/ f_prime(x_old)) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    return (x_old, normdiff, iter)
end

maxiter = 1000
tolerance = 1.0E-7
x_0 = 0.0 # initial condition

f(v) = (v-1)^3
f_prime(v)= 3*(v-1)^2

x_star, normdiff, iter = Newtonfixedpointmap(f, f_prime, x_0, tolerance, maxiter)
println("Fixed point = $x_star, and |f(x) - x| = $normdiff in $iter iterations")

f(v)= 27*v^3 - 3*v + 1
f_prime(v)= 81*v^2 - 3

x_star, normdiff, iter = Newtonfixedpointmap(f, f_prime, x_0, tolerance, maxiter)
println("Fixed point = $x_star, and |f(x) - x| = $normdiff in $iter iterations")

using ForwardDiff

# operator to get the derivative of this function using AD
D(f) = v-> ForwardDiff.derivative(f, v)

function Newtonfixedpointmap(f, x_0, tolerance, maxiter)
    # setup the algorithm
    x_old = x_0
    normdiff = Inf
    iter = 1
    f_prime = D(f)
    while normdiff > tolerance && iter <= maxiter
        x_new = x_old - (f(x_old)/ f_prime(x_old)) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    return (x_old, normdiff, iter)
end
maxiter = 1000
tolerance = 1.0E-7
x_0 = 0.0 # initial condition

f(v) = (v-1)^3
f_prime(v)= 3*(v-1)^2

x_star, normdiff, iter = Newtonfixedpointmap(f, f_prime, x_0, tolerance, maxiter)
println("Fixed point = $x_star, and |f(x) - x| = $normdiff in $iter iterations")

f(v)= 27*v^3 - 3*v + 1
f_prime(v)= 81*v^2 - 3

x_star, normdiff, iter = Newtonfixedpointmap(f, f_prime, x_0, tolerance, maxiter)
println("Fixed point = $x_star, and |f(x) - x| = $normdiff in $iter iterations")


# # Time Series Analysis
# A time series is a sequence of data points, each associated with a time. In our example, we will work with a time series of daily temperatures in the city of Melbourne, Australia over a period of a few years. Let $x$ be the vector of the time series, and $x_i$ denote the temperature in Melbourne on day $i$. Here is a picture of the time series:

using Plots, Convex, ECOS, DelimitedFiles
aux(str) = joinpath(@__DIR__, "aux_files", str) # path to auxiliary files

temps = readdlm(aux("melbourne_temps.txt"), ',')
n = size(temps, 1)
plot(1:n, temps[1:n], ylabel="Temperature (°C)", label="data", xlabel = "Time (days)", xticks=0:365:n)

# We can quickly compute the mean of the time series to be $11.2$. If we were to always guess the mean as the temperature of Melbourne on a given day, the RMS error of our guesswork would be $4.1$. We'll try to lower this RMS error by coming up with better ways to model the temperature than guessing the mean.
#
# A simple way to model this time series would be to find a smooth curve that approximates the yearly ups and downs.
# We can represent this model as a vector $s$ where $s_i$ denotes the temperature on the $i$-th day.
# To force this trend to repeat yearly, we simply want
#
# $$
#  s_i = s_{i + 365}
# $$
#
# for each applicable $i$.
#
# We also want our model to have two more properties:
#
# - The first is that the temperature on each day in our model should be relatively close to the actual temperature of that day.
# - The second is that our model needs to be smooth, so the change in temperature from day to day should be relatively small. The following objective would capture both properties:
#
# $$
#  \sum_{i = 1}^n (s_i - x_i)^2 + \lambda \sum_{i = 2}^n(s_i - s_{i - 1})^2
# $$
#
# where $\lambda$ is the smoothing parameter. The larger $\lambda$ is, the smoother our model will be.
#
# The following code uses Convex to find and plot the model:

yearly = Variable(n)
eq_constraints = [ yearly[i] == yearly[i - 365] for i in 365 + 1 : n ]

smoothing = 100
smooth_objective = sumsquares(yearly[1 : n - 1] - yearly[2 : n])
problem = minimize(sumsquares(temps - yearly) + smoothing * smooth_objective, eq_constraints);
solve!(problem, () -> ECOS.Optimizer(maxit=200, verbose=0))
residuals = temps - evaluate(yearly)

## Plot smooth fit
plot(1:n, temps[1:n], label="data")
plot!(1:n, evaluate(yearly)[1:n], linewidth=2, label="smooth fit",  ylabel="Temperature (°C)", xticks=0:365:n, xlabel="Time (days)")

# We can also plot the residual temperatures, $r$, defined as $r = x - s$.

## Plot residuals for a few days
plot(1:100, residuals[1:100], ylabel="Residuals", xlabel="Time (days)")

#-

root_mean_square_error = sqrt(sum( x -> x^2, residuals) / length(residuals))
# Our smooth model has a RMS error of $2.7$, a significant improvement from just guessing the mean, but we can do better.
#
# We now make the hypothesis that the residual temperature on a given day is some linear combination of the previous $5$ days. Such a model is called autoregressive. We are essentially trying to fit the residuals as a function of other parts of the data itself. We want to find a vector of coefficients $a$ such that
#
# $$
#  \text{r}(i) \approx \sum_{j = 1}^5 a_j \text{r}(i - j)
# $$
#
# This can be done by simply minimizing the following sum of squares objective
#
# $$
#  \sum_{i = 6}^n \left(\text{r}(i) - \sum_{j = 1}^5 a_j \text{r}(i - j)\right)^2
# $$
#
# The following Convex code solves this problem and plots our autoregressive model against the actual residual temperatures:

## Generate the residuals matrix
ar_len = 5

residuals_mat = Matrix{Float64}(undef, length(residuals) - ar_len, ar_len)
for i = 1:ar_len
  residuals_mat[:, i] = residuals[ar_len - i + 1 : n - i]
end

## Solve autoregressive problem
ar_coef = Variable(ar_len)
problem = minimize(sumsquares(residuals_mat * ar_coef - residuals[ar_len + 1 : end]))
solve!(problem, () -> ECOS.Optimizer(max_iters=200, verbose=0))

## plot autoregressive fit of daily fluctuations for a few days
ar_range = 1:145
day_range = ar_range .+ ar_len
plot(day_range, residuals[day_range], label="fluctuations from smooth fit", ylabel="Temperature difference (°C)")
plot!(day_range, residuals_mat[ar_range, :] * evaluate(ar_coef), label="autoregressive estimate", xlabel="Time (days)")

# Now, we can add our autoregressive model for the residual temperatures to our smooth model to get an better fitting model for the daily temperatures in the city of Melbourne:

total_estimate = evaluate(yearly)
total_estimate[ar_len + 1 : end] += residuals_mat * evaluate(ar_coef)

# We can plot the final fit of data across the whole time range:
plot(1:n, temps, label="data", ylabel="Temperature (°C)")
plot!(1:n, total_estimate, label="estimate", xticks=0:365:n, xlabel="Time (days)")

# The RMS error of this final model is $\sim 2.3$:
root_mean_square_error = sqrt(sum( x -> x^2, total_estimate - temps) / length(temps))


# numerial integration trapezoidal method

  function integration(a,b,n::Int64, func::Function)
    sum = 0.
    step = (b-a) / n # b and a are, respectivelly, upper and lower limits. n is the number of steps
    fa = func(a) / 2
    fb = func(b) / 2
    for i in 1:n-1
      x = i*step + a
      sum += func(x)
    end
    sum = (sum + fb + fa)*step

    return sum
  end

# Numerical integration

  function width(array)
    dt = ((maximum(array) - minimum(array))/(length(array)-1))
    return dt
  end

# NUMERICAL INTEGRATION - TRAPEZIUM METHOD

  function integration(x::Vector, y::Vector)
    dx = width(x)

# I = (h/2) * (y(1) + 2*y(2) + ... + 2y(n) + y(n+1)
    I = 0.
    n = length(y) - 1
    for i in 2:n
      I = I + y[i]
    end
    I = (dx/2)*(y[1]+2*I+y[n+1])

    return I

  end

# Simpson's method

  function integration_simp(x::Vector, y::Vector)
    dx = width(x)
    n = length(y) - 1

    if n % 2 != 0
      n += 1
    end

    I = y[1]
    for i in 2:n
      if i % 2 == 0
        I = I + 4*y[i]
      elseif i % 2 == 1
        I = I + 2*y[i]
      end
    end

    I = (dx/3) * (I + y[n+1])
    return I
  end


using LinearAlgebra, Quaternions, ReferenceFrameRotations
# Check if a given vector is three-dimensional
function check_vector_3(v)
    if size(v) != (1,3) && size(v) != (3, 1) && size(v) != (3,)
        println(size(v))
        println("\nIncorrect vector3. You must introduce an R3 vector.")
        return false
    end
end
# Format vector3 to be vertically oriented
function format_vector_3(v)
    if size(v) == (1,3)
        v = v'
    end
    return v
end

function to_2d(vector3)
    # Check if the given vector is actually on three dimensions
    if check_vector_3(vector3) == false return end
    # If the vector is given horizontal, we transpose it
    vector3 = format_vector_3(vector3)

    # Defined matrix to convert R3 vector to R2 vector
    # (we find this matrix on the .pdf attached file)
    axonometry_matrix = [-0.5*cosd(42)    cosd(7)     0;
                       -0.5*sind(42)    -sind(7)    1]

    #=axonometry_matrix = [-cosd(30)    cosd(30)     0;
                        -sind(30)    -sind(30)    1]=#

    # Return the result of the multiplication
    return axonometry_matrix * vector3
end

function rotate_phi_z(X, phi)
    # Check if the given vector is actually on three dimensions
    if check_vector_3(X) == false
        return
    end
    # If the vector is given horizontal, we transpose it
    X = format_vector_3(X)

    # Define rotation matrix on Z axis
    R = [ 0 -1 0;
          1 0 0;
          0 0 0 ]

    # Apply the Rodrigues formula
    result = I + sind(phi) * R + (1 - cosd(phi)) * R^2

    # Calculate result using the rotation matrix
    T = result * X
    return T

end

function rotate_phi(X, phi, axis)
    # Check if the given vector is actually on three dimensions
    if check_vector_3(X) == false
        return
    end
    # If the vector is given horizontal, we transpose it
    X = format_vector_3(X)

    axis = normalize(axis)

    # Define rotation matrix on Z axis
    R = [ 0      -axis[3]    axis[2];
         axis[3]    0      -axis[1];
         -axis[2]  axis[1]      0 ]

    # Apply the Rodrigues formula
    result = I + sind(phi) * R + (1 - cosd(phi)) * R^2

    # Calculate result using the rotation matrix
    T = result * X
    return T

end

# To check if the given vector is of dimension 3
function check_vector_3(v)
    if size(v) != (1,3) && size(v) != (3, 1) && size(v) != (3,)
        print("Your vector is size ", size(v))
        println("\nIncorrect vector3. You must introduce an R3 vector.")
        return false
    end
end

# To check if the given object is a Quaternion
function check_quaternion(q)
    if typeof(q) == Quaternions.Quaternion{Float64} || typeof(q) == Quaternions.Quaternion{Int64}
        return true
    end
    println("You must introduce a valid Quaternion! Using Quaternion()")
    return false
end

# To check if the given matrix is of dimensions 3x3
function check_matrix_3(R)
    if size(R) != (3,3)
        print("Your matrix is size ", size(R))
        println("\nIncorrect matrix. You must introduce a 3x3 matrix.")
        return false
    end
end

function check_euler_angleaxis(E)
    if typeof(E) == EulerAngleAxis{Float64} || typeof(E) == EulerAngleAxis{Int64}
        return true
    end
    println("Incorrect EulerAngleAxis. You must introduce an EulerAngleAxis()")
    return false
end
# Axis/Angle to Quaternion
function axis_angle_to_quat(E)

    if check_euler_angleaxis(E) == false
        return
    end

    # We get the axis from our EulerAngleAxis
    V = Vector{Float64}(E.v)
    # We get the angle from our EulerAngleAxis
    phi = E.a

    # Use qrotation function to create a Quaternion with the given axis and angle
    qR = qrotation(V, phi)

    return qR

end

function axis_angle_to_mat(E)

    if check_euler_angleaxis(E) == false
        return
    end
    # We get the axis from our EulerAngleAxis
    V = normalize(E.v)
    # We get the angle from our EulerAngleAxis
    phi = E.a

    # We use Rodrigues Formula to find the Rotation matrix
    # Define rotation matrix on Z axis
    R = [ 0     -V[3]   V[2];
          V[3]   0      -V[1];
         -V[2]   V[1]      0 ]

    # Apply the Rodrigues formula
    result = I + sin(phi) * R + (1 - cos(phi)) * R^2

    return result
end

function mat_to_axis_angle(R)
    if check_matrix_3(R) == false
        return
    end

# Find angle
phi = acos((tr(R) -1) / 2)

# Find axis matrix
V = (R - R') / (2 * sin(phi))

# Get axis values from the axis matrix into a vector
axis = [V[3,2]; V[1,3]; V[2,1]]

return EulerAngleAxis(phi, axis)
end

function quat_to_axis_angle(q)
    if check_quaternion(q) == false
        return
    end

    # Get the angle
    angle = 2 * atan(sqrt(q.v1^2 + q.v2^2 + q.v3^2) / q.s)

    # Normalize the Quaternion
    q = normalize(q)

    # We do the same process as in the
    s = sin(angle/2)

    # If the s is not zero, we divide. If it is zero, we can't divide.
    # Therefore, we set axis as the [1 0 0] vector
    s != 0 ? axis = [q.v1; q.v2; q.v3] / s : axis = [1.0; 0.0; 0.0]

    return EulerAngleAxis(angle, axis)
end

function quat_to_mat(q)
    if check_quaternion(q) == false
        return
    end
    # We apply the matrix to Quaternion transformation
    R = [q.s^2 + q.v1^2 - q.v2^2 - q.v3^2    2*q.v1*q.v2 - 2*q.s*q.v3            2*q.v1*q.v3 + 2*q.s*q.v2;
     2*q.v1*q.v2 + 2*q.s*q.v3                q.s^2 - q.v1^2 + q.v2^2 - q.v3^2    2*q.v2*q.v3 - 2*q.s*q.v1;
     2*q.v1*q.v3 - 2*q.s*q.v2                2*q.v2*q.v3 + 2*q.s*q.v1            q.s^2 - q.v1^2 - q.v2^2 + q.v3^2]

     return R

end

function mat_to_quat(R)
    if check_matrix_3(R) == false
        return
    end

    E = mat_to_axis_angle(R)

    q = axis_angle_to_quat(E)

    return q
end

function rescale_transalte_vector(V, pos, scale)
    # Get Vector without considering origin
    # Change Vector position using pos
    # Position is on world coordinates (always positive)

    #Check vector and position are on the correct format
    V = format_vector_3(V)
    pos = format_vector_3(pos)

    # Rescale vector
    V = V*scale

    # Change vector position
    V = V+pos

    return V

end


# adapted from https://www.youtube.com/watch?v=vAp6nUMrKYg
struct Dual <: Number
    x::Float64
    y::Float64
end

import Base: +, -, *, /, convert, promote_rule, show

+(a::Dual, b::Dual) = Dual(a.x+b.x, a.y+b.y)
-(a::Dual, b::Dual) = Dual(a.x-b.x, a.y-b.y)

*(a::Dual, b::Dual) = Dual(a.x*b.x, a.y*b.x + a.x*b.y)
/(a::Dual, b::Dual) = Dual(a.x/b.x, (a.y*b.x - a.x*b.y)/(b.x^2))

convert(::Type{Dual}, x::Real) = Dual(x, zero(x))
promote_rule(::Type{Dual}, ::Type{<:Number}) = Dual
show(io::IO, d::Dual) = print(io, d.x, " + ", d.y," ϵ")

f(x) = x^2
f(1)
f(1.0)
f(Dual(1,1))

"""
Compute the square root of x using the Babylonian algorithm
"""
function babylonian(x; nmax = 10)
    t = (1+x)/2
    for i in 2:nmax
        t = (t + x/t)/2
    end
    return t
end

babylonian(π), √π
babylonian(2), √2

# What happens when we pass a Dual number to babylonian?
x = Dual(2,1)
# We automagically get the derivative!
babylonian(x)
√2, 0.5/√2


function nonLinChem(dy,y,p,t)
 dy[1] = -y[1]
 dy[2] = y[1]-(y[2])^2
 dy[3] = (y[2])^2
end
y0 = [1.0;0.0;0.0]
tspan = (0.0,20.0)
nlc_analytic(u0,p,t) = [exp(-t);
    (2sqrt(exp(-t))besselk(1,2sqrt(exp(-t)))-2besselk(1,2)/besseli(1,2)*sqrt(exp(-t))besseli(1,2sqrt(exp(-t))))/(2besselk(0,2sqrt(exp(-t)))+(2besselk(1,2)/besseli(1,2))besseli(0,2sqrt(exp(-t))));
    -exp(-t)+1+(-2sqrt(exp(-t))*besselk(1,2sqrt(exp(-t)))+sqrt(exp(-t))*besseli(1,2sqrt(exp(-t)))*2besselk(1,2)/besseli(1,2))/(2besselk(0,2sqrt(exp(-t)))+2besselk(1,2)/besseli(1,2)*besseli(0,2sqrt(exp(-t))))]
nonLinChem_f = ODEFunction(nonLinChem,analytic = nlc_analytic)

@doc doc"""
Nonlinear system of reactions with an analytical solution

```math
\frac{dy_1}{dt} = -y_1
```

```math
\frac{dy_2}{dt} = y_1 - y_2^2
```

```math
\frac{dy_3}{dt} = y_2^2
```

with initial condition ``y=[1;0;0]`` on a time span of ``t \in (0,20)``

From

Liu, L. C., Tian, B., Xue, Y. S., Wang, M., & Liu, W. J. (2012). Analytic solution
for a nonlinear chemistry system of ordinary differential equations. Nonlinear
Dynamics, 68(1-2), 17-21.

The analytical solution is implemented, allowing easy testing of ODE solvers.
"""
prob_ode_nonlinchem = ODEProblem(nonLinChem,y0,tspan)


using LinearAlgebra
using Symbolics

# Problem One

A = Matrix{Int8}([0 5 0; 8 3 7; 9 -2 9])
B = Matrix{Int8}([4 6 -2; 7 2 3; 1 3 -4])
C = Vector{Int8}([-1, 2, 5])

println("Result of A*B: ")
display(A*B)
println("")

println("Result of A*C: ")
display(A*C)
println("")

println("Identity Matrix of Matrix A: ")
identityA = Matrix{Int8}(I,3,3)
display(identityA)
println("")

println("Result of I*A: ")
display(identityA*A)
println("")

println("Result of A*I: ")
display(A*identityA)
println("")

println("Matrix A")
display(A)
println("")

#Problem Two

A = Matrix{Int8}([5  7  2  0  3  5;
     3  8 -3 -5  0  8;
     1  4  0  7 15  9;
     0 10  5 12  3 -1;
     2 -5  9  2 18 -10])

B = Matrix{Int8}([ 2 10  0;
      8  7  5;
     -5  2 -4;
      4  8 13;
      3 12  0;
      1  5  7])

println("Result C = A*B: ")
display(A*B)
println("")

println("Transpose of Matrix A: ")
display(transpose(A))
println("")

# Problem Three

A = Matrix{Int8}([1 5 0; 8 3 7; 6 -2 9])

println("Determinant of Matrix A: ")
display(det(A))
println("")

println("Determinant of the Transpose of Matrix A: ")
display(det(transpose(A)))
println("")

println("Inverse of Matrix A: ")
display(inv(A))
println("")

# Problem Four
@variables k1
A = [0 0 0 0;
     0 k1 0 -k1;
     0 0 0 0;
     0 -k1 0 k1;]
println("Local Matrix A: ")
display(A)
println("")

G = [0 0 0 0 0 0 0 0 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 k1 0 0 0 -k1 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 -k1 0 0 0 k1 0 0;
     0 0 0 0 0 0 0 0 0 0;
     0 0 0 0 0 0 0 0 0 0;]
println("Mapped global matrix, G: ")
display(G)


#############################################################
function mylogpdf_Cov(x, μ, Σ)
#############################################################

    @assert(length(x) == length(μ) == size(Σ,1) == size(Σ,2))

    d = length(μ)

    L = chol(Hermitian(Σ))'

    TERM1 = -0.5 * 2.0*sum(log.(diag(L))) # consult eq. A.18 in RW
    TERM2 = -0.5 * d * log(2π)
    TERM3 = -0.5 * sum(( (L\(x-μ)).^2 ))

    return TERM1 + TERM2 + TERM3

end

#############################################################
function mylogpdf_scalCov(x, μ, σ2)
#############################################################

    @assert(length(x) == length(μ))

    d = length(μ)

    TERM1 = -0.5 * d * log(σ2)
    TERM2 = -0.5 * d * log(2π)
    TERM3 = -0.5 * sum((x-μ).^2)/σ2

    return TERM1 + TERM2 + TERM3

end


### A Pluto.jl notebook ###
# v0.16.0

using Markdown
using InteractiveUtils

# ╔═╡ 863debb0-fc11-11ea-09e6-dfb25ad73b07
using JuMP

# ╔═╡ 894add90-fc11-11ea-16ed-0120c97d8aec
using GLPK

# ╔═╡ 6f96aa50-fc11-11ea-22c9-01e5e73d8ab0
md"# Applications of Linear Programming"

# ╔═╡ 8c3d90b0-fc11-11ea-2873-934ec496d1ee
md"""## Economy

A manufacturer produces  four different  products  $X_1$, $X_2$, $X_3$ and $X_4$. There are three inputs to this production process:

- labor in man weeks,
- kilograms of raw material A, and
- boxes  of raw  material  B.

Each product has different input requirements. In determining each  week's production schedule, the manufacturer cannot use more than the available amounts of  manpower and the two raw  materials:

|Inputs|$X_1$|$X_2$|$X_3$|$X_4$|Availabilities|
|------|-----|-----|-----|-----|--------------|
|Person-weeks|1|2|1|2|20|
|Kilograms of material A|6|5|3|2|100|
|Boxes of material B|3|4|9|12|75|
|Production level|$x_1$|$x_2$|$x_3$|$x_4$| |

These constraints can be written in mathematical form

```math
\begin{aligned}
x_1+2x_2+x_3+2x_4\le&20\\
6x_1+5x_2+3x_3+2x_4\le&100\\
3x_1+4x_2+9x_3+12x_4\le&75
\end{aligned}
```

Because negative production levels are not meaningful, we must impose the following nonnegativity constraints on the production levels:

```math
x_i\ge0,\qquad i=1,2,3,4
```

Now suppose that one unit of product $X_1$ sells for €6 and $X_2$, $X_3$ and $X_4$ sell for €4, €7 and €5, respectively. Then, the total revenue for any production decision $\left(x_1,x_2,x_3,x_4\right)$ is

```math
f\left(x_1,x_2,x_3,x_4\right)=6x_1+4x_2+7x_3+5x_4
```

The problem is then to maximize $f$ subject to the given constraints."""

# ╔═╡ acba3730-fc11-11ea-2d31-27432ab5e777
let
	model = Model(GLPK.Optimizer)
	@variable(model, 0 <= x1)
	@variable(model, 0 <= x2)
	@variable(model, 0 <= x3)
	@variable(model, 0 <= x4)
	@objective(model, Max, 6*x1 + 4*x2 + 7*x3 + 5*x4)
	@constraint(model, con1,   x1 + 2*x2 +   x3 +  2*x4 <= 20)
	@constraint(model, con2, 6*x1 + 5*x2 + 3*x3 +  2*x4 <= 100)
	@constraint(model, con3, 3*x1 + 4*x2 + 9*x3 + 12*x4 <= 75)
	optimize!(model)
	termination_status(model), primal_status(model), value(x1), value(x2), value(x3), value(x4), objective_value(model)
end

# ╔═╡ 06330800-fc12-11ea-011c-e762078d0d88
md"""## Manufacturing

A manufacturer produces two different products $X_1$ and $X_2$ using three machines $M_1$, $M_2$, and $M_3$. Each machine can be used only for a limited amount of time. Production times of each product on each machine are given by

|Machine|Production time $X_1$|Production time $X_2$|Available time|
|-------|---------------------|---------------------|--------------|
|$M_1$  |1                    |1                    |8             |
|$M_2$  |1                    |3                    |18            |
|$M_3$  |2                    |1                    |14            |
|Total  |4                    |5                    |              |

The objective is to maximize the combined time of utilization of all three machines.

Every production decision must satisfy the constraints on the available time. These restrictions can be written down using data from the table.

```math
\begin{aligned}
x_1+x_2&\le8\,,\\
x_1+3x_2&\le18\,,\\
2x_1+x_2&\le14\,,
\end{aligned}
```

where $x_1$ and $x_2$ denote the production levels. The combined production time of all three machines is

```math
f\left(x_1,x_2\right)=4x_1+5x_2\,.
```"""

# ╔═╡ 51553150-fc12-11ea-2020-7b418f5eb559
let
	model = Model(GLPK.Optimizer)
	@variable(model, 0 <= x1)
	@variable(model, 0 <= x2)
	@objective(model, Max, 4*x1 + 5*x2)
	@constraint(model, con1,   x1 +   x2 <= 8)
	@constraint(model, con2,   x1 + 3*x2 <= 18)
	@constraint(model, con3, 2*x1 +   x2 <= 14)
	optimize!(model)
	termination_status(model), primal_status(model), value(x1), value(x2), objective_value(model)
end

# ╔═╡ 7616fb90-fc12-11ea-1b4b-f7ad83c91fe9
md"""## Transportation

A manufacturing company has plants in cities A, B, and C. The company produces and distributes its product to dealers in various cities. On a particular day, the company has 30 units of its product in A, 40 in B, and 30 in C. The company plans to ship 20 units to D, 20 to E, 25 to F, and 35 to G, following orders received from dealers. The transportation costs per unit of each product between the cities are given by

|From|To D|To E|To F|To G|Supply|
|----|----|----|----|----|------|
|A   |7   |10  |14  |8   |30    |
|B   |7   |11  |12  |6   |40    |
|C   |5   |8   |15  |9   |30    |
|Demand|20|20  |25  |35  |100   |

In the table, the quantities supplied and demanded appear at the right and along the bottom of the table. The quantities to be transported from the plants to different destinations are represented by the decision variables.

This problem can be stated in the form:

```math
\min 7x_{AD}+10x_{AE}+14x_{AF}+8x_{AG}+7x_{BD}+11x_{BE}+12x_{BF}+6x_{BG}+5x_{CD}+8x_{CE}+15x_{CF}+9x_{CG}
```

subject to

```math
\begin{aligned}
x_{AD}+x_{AE}+x_{AF}+x_{AG}&=30\\
x_{BD}+x_{BE}+x_{BF}+x_{BG}&=40\\
x_{CD}+x_{CE}+x_{CF}+x_{CG}&=30\\
x_{AD}+x_{BD}+x_{CD}&=20\\
x_{AE}+x_{BE}+x_{CE}&=20\\
x_{AF}+x_{BF}+x_{CF}&=25\\
x_{AG}+x_{BG}+x_{CG}&=35
\end{aligned}
```

In this problem, one of the constraint equations is redundant because it can be derived from the rest of the constraint equations. The mathematical formulation of the transportation problem is then in a linear programming form with twelve (3x4) decision variables and six (3 + 4 - 1) linearly independent constraint equations. Obviously, we also require nonnegativity of the decision variables, since a negative shipment is impossible and does not have any valid interpretation."""

# ╔═╡ d6798340-fc12-11ea-0b8d-71931d797511
let
	model = Model(GLPK.Optimizer)
	@variable(model, 0 <= x[1:3,1:4])
	@objective(model, Min, 7x[1,1]+10x[1,2]+14x[1,3]+8x[1,4]+7x[2,1]+11x[2,2]+12x[2,3]+6x[2,4]+5x[3,1]+8x[3,2]+15x[3,3]+9x[3,4])
	@constraint(model, con1, sum(x[1,j] for j in 1:4) == 30)
	@constraint(model, con2, sum(x[2,j] for j in 1:4) == 40)
	@constraint(model, con3, sum(x[3,j] for j in 1:4) == 30)
	@constraint(model, con4, sum(x[i,1] for i in 1:3) == 20)
	@constraint(model, con5, sum(x[i,2] for i in 1:3) == 20)
	@constraint(model, con6, sum(x[i,3] for i in 1:3) == 25)
	@constraint(model, con7, sum(x[i,4] for i in 1:3) == 35)
	optimize!(model)
	termination_status(model), primal_status(model), value.(x), objective_value(model)
end

# ╔═╡ 6d012070-fc13-11ea-1f08-b3754cf9c6bd
md"""This problem is an _integer linear programming_ problem, i.e. the solution components must be integers.

We can use the simplex method to find a solution to an ILP problem if the $m\times n$ matrix $A$ is unimodular, i.e. if all its nonzero $m$th order minors are $\pm 1$."""

# ╔═╡ 7a077a30-fc13-11ea-193e-6b30f852c9e7
md"""## Electricity

An electric circuit is designed to use a 30 V source to charge 10 V, 6 V, and 20 V batteries connected in parallel. Physical constraints limit the currents $I_1$, $I_2$, $I_3$, $I_4$, and $I_5$ to a maximum of 4 A, 3 A, 3 A, 2 A, and 2 A, respectively. In addition, the batteries must not be discharged, that is, the currents $I_1$, $I_2$, $I_3$, $I_4$, and $I_5$ must not be negative. We wish to find the values of the currents $I_1$, $I_2$, $I_3$, $I_4$, and $I_5$ such that the total power transferred to the batteries is maximized.

The total power transferred to the batteries is the sum of the powers transferred to each battery, and is given by $10I_2 + 6I_4 + 20I_5$ W. From the circuit, we observe that the currents satisfy the constraints $I_1 = I_2 + I_3$, and $I_3 = I_4 + I_5$. Therefore, the problem can be posed as the following linear program:

```math
\max 10I_2+6I_4+20I_5
```

subject to

```math
\begin{aligned}
I_1 &= I_2 + I_3\\
I_3 &= I_4 + I_5\\
I_1 &\le 4\\
I_2 &\le 3\\
I_3 &\le 3\\
I_4 &\le 2\\
I_5 &\le 2
\end{aligned}
```

"""

# ╔═╡ 98561d20-fc13-11ea-0278-cf26118a2c51
let
	model = Model(GLPK.Optimizer)
	@variable(model, 0 <= I[1:5])
	@objective(model, Max, 10*I[2]+6*I[4]+20I[5])
	@constraint(model, con1, I[1] == I[2] + I[3])
	@constraint(model, con2, I[3] == I[4] + I[5])
	@constraint(model, con3, I[1] <= 4)
	@constraint(model, con4, I[2] <= 3)
	@constraint(model, con5, I[3] <= 3)
	@constraint(model, con6, I[4] <= 2)
	@constraint(model, con7, I[5] <= 2)
	optimize!(model)
	termination_status(model), primal_status(model), value.(I), objective_value(model)
end

# ╔═╡ 478fe6e0-fc14-11ea-10d4-4d9e0de6ea00
md"""## Telecom

Consider a wireless communication system. There are $n$ "mobile" users. For each $i$ in $1,\dots, n$; user $i$ transmits a signal to the base station with power $P_i$ and an attenuation factor of $h_i$ (i.e., the actual received signal power at the basestation from user $i$ is $h_iP_i$). When the basestation is receiving from user $i$, the total received power from all other users is considered "interference" (i.e., the interference for user $i$ is $\sum_{i\ne j}h_jP_j$). For the communication with user $i$ to be reliable, the signal-to-interference ratio must exceed a threshold $\gamma_i$, where the "signal" is the received power for user $i$.

We are interested in minimizing the total power transmitted by all the users subject to having reliable communications for all users. We can formulate the problem as a linear programming problem of the form

```math
\min \sum_iP_i
```

subject to

```math
\forall i \in 1,\dots,n\,:\,\begin{cases}
\frac{h_iP_i}{\sum_{i\ne j}h_jP_j}\ge\gamma_i\\
P_i\ge0
\end{cases}
```"""

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
GLPK = "60bf3e95-4087-53dc-ae20-288a0d20c6a6"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"

[compat]
GLPK = "~0.14.14"
JuMP = "~0.21.10"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[BenchmarkTools]]
deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
git-tree-sha1 = "61adeb0823084487000600ef8b1c00cc2474cd47"
uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
version = "1.2.0"

[[BinaryProvider]]
deps = ["Libdl", "Logging", "SHA"]
git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
version = "0.5.10"

[[Bzip2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
version = "1.0.8+0"

[[CEnum]]
git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
version = "0.4.1"

[[Calculus]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad"
uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9"
version = "0.5.1"

[[ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "4ce9393e871aca86cc457d9f66976c3da6902ea7"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.4.0"

[[CodecBzip2]]
deps = ["Bzip2_jll", "Libdl", "TranscodingStreams"]
git-tree-sha1 = "2e62a725210ce3c3c2e1a3080190e7ca491f18d7"
uuid = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
version = "0.7.2"

[[CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.0"

[[CommonSubexpressions]]
deps = ["MacroTools", "Test"]
git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
version = "0.3.0"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "4866e381721b30fac8dda4c8cb1d9db45c8d2994"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.37.0"

[[CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.10"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[DiffResults]]
deps = ["StaticArrays"]
git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805"
uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
version = "1.0.3"

[[DiffRules]]
deps = ["NaNMath", "Random", "SpecialFunctions"]
git-tree-sha1 = "7220bc21c33e990c14f4a9a319b1d242ebc5b269"
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
version = "1.3.1"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.5"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "b5e930ac60b613ef3406da6d4f42c35d8dc51419"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
version = "0.10.19"

[[GLPK]]
deps = ["BinaryProvider", "CEnum", "GLPK_jll", "Libdl", "MathOptInterface"]
git-tree-sha1 = "833dbc8fbb0554e31186df509d67fc2f78f1bb09"
uuid = "60bf3e95-4087-53dc-ae20-288a0d20c6a6"
version = "0.14.14"

[[GLPK_jll]]
deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "01de09b070d4b8e3e1250c6542e16ed5cad45321"
uuid = "e8aa6df9-e6ca-548a-97ff-1f85fc5b8b98"
version = "5.0.0+0"

[[GMP_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
git-tree-sha1 = "c7ec02c4c6a039a98a15f955462cd7aea5df4508"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.8.19"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[IrrationalConstants]]
git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.0"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[JSONSchema]]
deps = ["HTTP", "JSON", "ZipFile"]
git-tree-sha1 = "b84ab8139afde82c7c65ba2b792fe12e01dd7307"
uuid = "7d188eb4-7ad8-530c-ae41-71a32a6d4692"
version = "0.3.3"

[[JuMP]]
deps = ["Calculus", "DataStructures", "ForwardDiff", "JSON", "LinearAlgebra", "MathOptInterface", "MutableArithmetics", "NaNMath", "Printf", "Random", "SparseArrays", "SpecialFunctions", "Statistics"]
git-tree-sha1 = "4358b7cbf2db36596bdbbe3becc6b9d87e4eb8f5"
uuid = "4076af6c-e467-56ae-b986-b466b2749572"
version = "0.21.10"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
deps = ["ChainRulesCore", "DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "34dc30f868e368f8a17b728a1238f3fcda43931a"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.3"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.8"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MathOptInterface]]
deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "JSON", "JSONSchema", "LinearAlgebra", "MutableArithmetics", "OrderedCollections", "SparseArrays", "Test", "Unicode"]
git-tree-sha1 = "575644e3c05b258250bb599e57cf73bbf1062901"
uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
version = "0.9.22"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.3"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[MutableArithmetics]]
deps = ["LinearAlgebra", "SparseArrays", "Test"]
git-tree-sha1 = "3927848ccebcc165952dc0d9ac9aa274a87bfe01"
uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0"
version = "0.2.20"

[[NaNMath]]
git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.5"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.5+0"

[[OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "438d35d2d95ae2c5e8780b330592b6de8494e779"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.3"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[Profile]]
deps = ["Printf"]
uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
git-tree-sha1 = "a322a9493e49c5f3a10b50df3aedaf1cdb3244b7"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "1.6.1"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "3240808c6d463ac46f1c1cd7638375cd22abbccb"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.2.12"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.6"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[ZipFile]]
deps = ["Libdl", "Printf", "Zlib_jll"]
git-tree-sha1 = "3593e69e469d2111389a9bd06bac1f3d730ac6de"
uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
version = "0.9.4"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╟─6f96aa50-fc11-11ea-22c9-01e5e73d8ab0
# ╠═863debb0-fc11-11ea-09e6-dfb25ad73b07
# ╠═894add90-fc11-11ea-16ed-0120c97d8aec
# ╟─8c3d90b0-fc11-11ea-2873-934ec496d1ee
# ╠═acba3730-fc11-11ea-2d31-27432ab5e777
# ╟─06330800-fc12-11ea-011c-e762078d0d88
# ╠═51553150-fc12-11ea-2020-7b418f5eb559
# ╟─7616fb90-fc12-11ea-1b4b-f7ad83c91fe9
# ╠═d6798340-fc12-11ea-0b8d-71931d797511
# ╟─6d012070-fc13-11ea-1f08-b3754cf9c6bd
# ╟─7a077a30-fc13-11ea-193e-6b30f852c9e7
# ╠═98561d20-fc13-11ea-0278-cf26118a2c51
# ╟─478fe6e0-fc14-11ea-10d4-4d9e0de6ea00
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


module MaxCut

using LinearAlgebra
using Convex
using SCS

"The Goemans-Williamson algorithm for the MAXCUT problem."

function maxcut(W::Matrix{<:Real}; iter::Int=100, tol::Real=0)
	"Partition a graph into two disjoint sets such that the sum of the edge weights
	which cross the partition is as large as possible (known to be NP-hard)."

	"A cut of a graph can be produced by assigning either 1 or -1 to each vertex. The Goemans-Williamson
	algorithm relaxes this binary condition to allow for vector assignments drawn from the (n-1)-sphere
	(choosing an n-1 dimensional space will ensure seperability). This relaxation can then be written as
	a semidefinite program. Once the optimal vector assignments are found, origin centered hyperplanes are generated and
	their corresponding cuts evaluated. After 'iter' trials, or when the desired tolerance is reached,
	the hyperplane with the highest corresponding binary cut is used to partition the vertices."

	"W:		Adjacency matrix."
	"tol:	Maximum acceptable distance between a cut and the MAXCUT upper bound."
	"iter:	Maximum number of hyperplane iterations before a cut is chosen."

	LinearAlgebra.checksquare(W)
	issymmetric(W)					|| throw(ArgumentError("Adjacency matrix must be symmetric."))
	all(W .>= 0)					|| throw(ArgumentError("Adjacency matrix must be nonnegative."))
	all(iszero.(diag(W)))			|| throw(ArgumentError("Diagonal of adjacency matrix must be zero (no self loops)."))
	(tol >= 0)						|| throw(ArgumentError("The tolerance must be nonnegative."))
	(iter > 0)						|| throw(ArgumentError("The number of iterations must be a positive integer."))

	"This is the standard SDP Relaxation of the MAXCUT problem, a reference can be found at,
	http://www.sfu.ca/~mdevos/notes/semidef/GW.pdf"
	k = size(W, 1)
	S = Semidefinite(k)

	expr = dot(W, S)
	constr = [S[i,i] == 1.0 for i in 1:k]
	problem = minimize(expr, constr...)
	solve!(problem, SCS.Optimizer(verbose=false))

	## ensure symmetric positive-definite
	A = 0.5 * (S.value + S.value')
	A += (max(0, -eigmin(A)) + 1e-14) * Matrix(I, size(A, 1), size(A, 1))

	X = Matrix(cholesky(A))

	## a non-trivial upper bound on MAXCUT
	upperbound = (sum(W) - dot(W, S.value)) / 4

	## random origin-centered hyperplanes, generated to produce partitions of the graph
	max_cut = -1
	max_partition = nothing

	for i in 1:iter
		gweval = X' * randn(k)
		partition = (findall(gweval .>= 0), findall(gweval .< 0))
		cut = sum(W[partition...])

		if cut > max_cut
			max_partition = partition
			max_cut = cut
		end

		(upperbound - max_cut < tol) && break
		(i == iter) && println("Max iterations reached.")
	end
	return max_cut, max_partition
end

export maxcut

end


function dlyap(A,Q)

	# Solving Discrete LYAPunov equation using Silvester equation
	#
	# Sylvester equation A*X + X*B + C = 0
	# if B = -(A')^(-1), C = Q*(A')^(-1)
	# => A*X - X*(A')^(-1) + Q*(A')^(-1) = 0
	# => A*X*A' - X + Q = 0  -> Discrete Lyapunov equation

	Atinv = inv(A')
	B = -Atinv
	C = Q*Atinv
	X = sylvester(A, B, C)

	return X

end

# ======================================================
function dlyap_test()

	# example from
	# Barraud, A.Y., "A numerical algorithm to solve A'XA - X = Q",
	# IEEE Trans. Auto. Contr., AC-22, pp. 883-885, 1977.

	A = [1.5 2.5 3.5 4.5 5.5 6.5 7.5;
		1 -1.5 2.5 3.5 4.5 5.5 6.5;
		0 0 1.5 -2.5 3.5 4.5 5.5;
		0 0 0 1.5 2.5 -3.5 4.5;
		0 0 0 1 1.5 2.5 -3.5;
		0 0 0 0 0 1.5 -2.5;
		0 0 0 0 0 1 1.5]

	q = [5.25 1.5 19.25 22.5 55.75 60 49.25;
		0 0 3.25 -0.25 -6 -11 11.25;
		0 0 62.75 83 183.25 222.75 129.75;
		0 0 0 56.5 208.25 232.25 250.75;
		0 0 0 0 532.25 739.5 476.75;
		0 0 0 0 0 631 745.75;
		0 0 0 0 0 0 133.75]

	Q = (q + q') - diagm(diag(q))

	x = [1 1 1 1 1 1 1;
		0 1 2 3 4 5 6;
		0 0 1 3 6 10 15;
		0 0 0 1 4 10 20;
		0 0 0 0 1 5 15;
		0 0 0 0 0 1 6;
		0 0 0 0 0 0 1];

	X = (x + x') - diagm(diag(x))

	A = A'
	Q = -Q

	X1 = dlyap(A,Q)

	return X,X1

end


# Unit 00 -  Brief Review of Vectors, Planes, and Optimization

using LinearAlgebra, Distributions

## Homework 0

### 4. Points and Vectors

x = [0.4, 0.3]
y = [-0.15, 0.2]

nx = norm(x)
ny = norm(y)

sqrt(0.4^2+0.3^2)
sqrt((-0.15)^2+0.2^2)

dotxy = dot(x,y)
0.4*-0.15+0.3*0.2

α = acos(0)

3.14/2

using LinearAlgebra
a = [4,1]
b = [2,3]
normC = dot(a,b)/norm(b)
c = (dot(a,b)/norm(b)^2) * b

### 7. Univariate Gaussians

#### Probability

# Direct approach
X = Normal(1,sqrt(2))
P = cdf(X,2) - cdf(X,0.5)

# Using Standard Normals
Z = Normal(0,1)
ZlBound = (0.5-1)/sqrt(2)
ZuBound = (2-1)/sqrt(2)
P = cdf(Z,ZuBound) - cdf(Z,ZlBound)

### 8. (Optional Ungraded Warmup) 1D Optimization via Calculus

using SymPy,Plots

x = symbols("x")

f = (1/3)* x^3 - x^2 - 3*x + 10
df = diff(f,x)
critPoints = solve(df,x)
df2 = diff(df,x)

df2_x1 = subs(df2, x => critPoints[1])
df2_x2 = subs(df2, x => critPoints[2])

f_x1 = subs(f, x => critPoints[1])
f_x2 = subs(f, x => critPoints[2])
f_m4 = subs(f, x => -4)
f_4 = subs(f, x => 4)
# or, more simply...
f(critPoints[1])
f(critPoints[2])
f(-4)
f(4)

plot(f,-4,4)
plot(-exp(-x))
plot(x^0.7)

### 9. Gradients and Optimization

surface((x,y) -> x^2+y^2)

using Plots
x=-10:0.1:10
y=-10:0.1:10
f2(x,y) = x^2+y^2
plot(x,y,f2,st=:surface,camera=(45,45))

### 10. Matrices and Vectors

A = [1 2 3; 4 5 6; 1 2 1]
B = [2 1 0; 1 4 4; 5 6 4]

rank(A)
rank(B)

### 12. Linear Independence, Subspaces and Dimension

A = [1 3; 2 6]

B = [1 2; 2 1]

C = [1 1 0; 0 1 1; 0 0 1]

D = [2 -1 -1; -1 2 -1; -1 -1 2]

rank(A)

rank(B)

rank(C)

rank(D)

### 13. Determinant

A = [1 2 3; 4 5 6; 1 2 1]

det(A)

det(A')

Unit 00 - Course Overview, Homework 0, Project 0/Unit 00 - Course Overview, Homework 0, Project 0.jl


# If we list all the natural numbers below 10 that are multiples of 3 or 5, we
# get 3, 5, 6 and 9. The sum of these multiples is 23.
#
# Find the sum of all the multiples of 3 or 5 below 1000.

using ProjectEulerSolutions

# Brute force approach, loop through checking for multiples.
function p001solution_loop(n::Integer=9)::Integer
    total_sum = 0
    @simd for i in 1:n
        if i % 3 == 0 || i % 5 == 0
            total_sum += i
        end
    end
    return total_sum
end

# Faster approach, sum of real numbers in a sequence is n*(n+1)/2, just scale
# and remove double counts.
function p001solution_closedform(n::Integer=9)::Integer
    n3  = div(n, 3)
    n5  = div(n, 5)
    n15 = div(n, 15)
    return  3 * div(n3 * (n3 + 1), 2) +
            5 * div(n5 * (n5 + 1), 2) -
           15 * div(n15 * (n15 + 1), 2)
end

p001 = Problems.Problem(Dict("Closed form" => p001solution_closedform,
                             "Loop" => p001solution_loop))

Problems.benchmark(p001, 999)


"""
    triangle_area(v1, v2, v3)

Area of a triangle defined by the vertices `v1`, `v2` and `v3`.
"""
function triangle_area(v1, v2, v3)
    u1 = v2 - v1
    u2 = v3 - v1

    return abs(u1[1]*u2[2] - u1[2]*u2[1])/2

end

"""
    isinsideconvex(p, vertices; tol=sqrt(eps()))

Determines if a point `p` is inside a convex region defined by the vertices in the array `vertices`.
"""
function isinsideconvex(p, vertices; tol=sqrt(eps()))
    n = length(vertices)

    center = [0.0, 0.0]
    for i=1:n
        center += vertices[i]
    end
    center = center/n

    convex_area = 0.0
    for i=1:n
        convex_area += triangle_area(center, vertices[i], vertices[mod1(i+1,n)])
    end

    test_area = 0.0
    for i=1:n
        test_area += triangle_area(p, vertices[i], vertices[mod1(i+1,n)])
    end

    return isapprox(test_area, convex_area, rtol = tol)
end


using Plots,DifferentialEquations,LaTeXStrings

##
V(x,y) = (1/2)*(x^2 + y^2 + 2*x^2*y - (2/3)*y^3)
x1 = y1 = range(-1,stop = 1, length = 1000)

##
V(x,y) = (1/2)*(x^2 + y^2 + 2*x^2*y - (2/3)*y^3)
x = y = range(-1,stop = 1, length = 1000)
contourf(x,y,V)
contour!(x,y,V,levels=0:1/6:1,title = "Henon-Heiles potential: Intensity map",linecolor = :white)
# savefig("potential_together.png")

##
function Hénon_Heiles(du,u,p,t)
    x  = u[1]
    y  = u[2]
    dx = u[3]
    dy = u[4]
    du[1] = dx
    du[2] = dy
    du[3] = -x - 2x*y
    du[4] = y^2 - y -x^2
end

plot()
e = 0.0833
for y in -0.2:0.1:0.4
    for ydot in -0.1:0.1:0.2
        # y = 0.02
        # ydot = -0.08
        xdot = sqrt(2*e - ydot^2 - y^2 + (2//3)*y^3)
        initial = [0.0,y,xdot,ydot]
        tspan = (0,500.)
        prob = ODEProblem(Hénon_Heiles, initial, tspan)
        sol = solve(prob, saveat = 0.001,Rodas5());
        Y = [i[2] for i in sol.u if (abs(i[1]) < 9.e-4)];
        Ydot = [i[4] for i in sol.u if (abs(i[1]) < 9.e-4)];
        scatter!(Y,Ydot,xlabel = "y",ylabel = "ẏ",label = false,title = "Poincare section: E = $(e)", titlefont=10,markersize = 2,markercolor = :black)
    end
end
plot!()
##
function Hénon_Heiles(du,u,p,t)
    x  = u[1]
    y  = u[2]
    dx = u[3]
    dy = u[4]
    du[1] = dx
    du[2] = dy
    du[3] = -x - 2x*y
    du[4] = y^2 - y -x^2
end
e = 0.125
y = 0.02
ydot = -0.08
xdot = sqrt(2*e - ydot^2 - y^2 + (2//3)*y^3)
initial = [0.0,y,xdot,ydot]
tspan = (0,500.)
prob = ODEProblem(Hénon_Heiles, initial, tspan)
sol = solve(prob, saveat = 0.001);
plot(sol,vars = (1,2,4))

##


function golden_method(f::Function, a::AbstractVector, b::AbstractVector;
                       tol=eps()*10, maxit=1000)
    α1 = (3 - sqrt(5)) / 2
    α2 = 1 - α1
    d = b - a
    x1 = a + α1*d
    x2 = a + α2*d
    s = ones(a)
    f1 = f(x1)
    f2 = f(x2)

    d = α1*α2*d
    it = 0

    while any(d .> tol) && it < maxit
        it += 1
        i = f2 .> f1
        x1[i] = x2[i]
        f1[i] = f2[i]
        d *= α2
        x2 = x1 + s.*(i- map(!, i)).*d
        s = sign.(x2 .- x1)
        f2 = f(x2)
    end

    it >= maxit && warn("`golden_method`: maximum iterations exceeded")

    i = f2 .> f1
    x1[i] = x2[i]
    f1[i] = f2[i]

    x1, f1
end

# function golden_method(f::Function, a::AbstractVector, b::AbstractVector;
#                        tol=eps()*10, maxit=1000)
#     out_x = similar(a)
#     out_f = similar(a)
#     N = length(a)
#     @inbounds for i=1:N
#         out_x[i], out_f[i] = golden_method(f, a[i], b[i]; tol=tol, maxit=maxit)
#     end
#     out_x, out_f
# end

"""
Applies Golden-section search to search for the _maximum_ of a function in
the interval (a, b)

https://en.wikipedia.org/wiki/Golden-section_search
"""
function golden_method(f::Function, a::Real, b::Real; tol::Float64=10*eps(),
                       maxit::Int=1000)
    α1 = (3 - sqrt(5)) / 2
    α2 = 1 - α1
    d = b - a
    x1 = a + α1*d
    x2 = a + α2*d
    s = 1.0
    f1 = f(x1)::Float64
    f2 = f(x2)::Float64

    d = α1*α2*d

    it = 0

    while d > tol && it < maxit
        it += 1

        d *= α2

        if f2 > f1
            x1 = x2
            f1 = f2
            x2 = x1 + s*d
        else
            x2 = x1 - s*d
        end

        s = sign(x2 - x1)

        f2 = f(x2)::Float64
    end

    it >= maxit && warn("`golden_method`: maximum iterations exceeded")

    if f2 > f1
        x1 = x2
        f1 = f2
    end

    x1, f1::Float64
end


#Generate 2D data from 4 spherical (σ=0.25) gaussian clusters centered at [-1,1], [1,-1], [-1,1] and [1,1]
"""
    generate_4_clusters(npts_per_cluster; σ=0.25)
    generate_4_clusters(::DataType, npts_per_cluster; σ=0.25)

Generate 2D data from 4 spherical gaussian clusters, with standard deviation `σ`, centered at [-1,1], [1,-1], [-1,1] and [1,1]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_4_clusters(T::DataType, npts_per_cluster; σ = 0.25)
    c = convert(Matrix{T},[1 1 -1 -1; 1 -1 1 -1])
    data = Matrix{T}(undef,2,4*npts_per_cluster)
    for i ∈ axes(c,2)
        rpts = randn(T,(2,npts_per_cluster))
        sindx = (i-1)*npts_per_cluster+1
        eindx = i*npts_per_cluster
        data[:,sindx:eindx] = σ*rpts .+ c[:,i]
    end
    return data
end

generate_4_clusters(npts_per_cluster; σ = 0.25) = generate_4_clusters(Float64, npts_per_cluster; σ)

#Generate 2D data arranged in a circle (default radius, r =0.5) around centres at [-1,1], [1,-1], [-1,1] and [1,1]
"""
    generate_4_circle_clusters(npts_per_cluster = 8; r = 0.5)
    generate_4_circle_clusters(::DataType, npts_per_cluster = 8; r = 0.5)

Generate 2D data arranged in a circle (default radius, r =0.5) around centres at [-1,1], [1,-1], [-1,1] and [1,1]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_4_circle_clusters(T::DataType, npts_per_cluster::Int = 8; r = 0.5)
    c = convert(Matrix{T},[1 1 -1 -1; 1 -1 1 -1])
    data = Matrix{T}(undef,2,4*npts_per_cluster)
    for i ∈ axes(c,2)
        sindx = (i-1)*npts_per_cluster+1
        eindx = i*npts_per_cluster
        pts = generate_circle_cluster(T, npts_per_cluster)
        data[:,sindx:eindx] = r*pts .+ c[:,i]
    end
    return data
end

generate_4_circle_clusters(npts_per_cluster::Int = 8; r = 0.5) = generate_4_circle_clusters(Float64, npts_per_cluster; r)

#Generate 2D data arranged in a circle of radius one around [0,0]
"""
    generate_circle_cluster(npts = 8)
    generate_circle_cluster(::DataType, npts = 8)

Generate 2D data arranged in a circle of radius one around [0,0]. `DataType` can be `Float32` or `Float64`, default is `Float64`.
"""
function generate_circle_cluster(T::DataType, npts::Int = 8)
    Δθ = 2π/npts
    θ = 0:Δθ:2π-Δθ
    x = Matrix{T}(undef,2,npts)
    for i ∈ eachindex(θ)
        x[1,i] = cos(θ[i])
        x[2,i] = sin(θ[i])
    end
    return x
end

generate_circle_cluster(npts::Int = 8) = generate_circle_cluster(Float64, npts)


A = [ 1. 2 3; 4 5 6; 7 8 9 ]

B = [12 -51 4; 6 167 -68; -4 24 -41]

function check_DV(A, D, V)
    I = eye(size(A)[1])
    for i = 1:size(A)[1]
        check = (A - D[i] * I) * V[:,i]
        println("For the ", i, "th eigen value and vector, (A - lambda I)^k * v = ", check, "\n\twhich has a L2 norm = ", norm(check))
        assert(norm(check) <= 1e-14)
    end
end

D_A, V_A = eig(A)

check_DV(A, D_A, V_A)

D_B, V_B = eig(B)

check_DV(B, D_B, V_B)

"Power iteration method on A, to find eigenpair with largest eigen value."
function poweriteration(A, maxiter=20, userandomstart=true)
    n = size(A)[1]
    if userandomstart
        X = randn((n, 1))
    else
        X = ones((n, 1))
    end
    for i = 1:maxiter
        next_X = A * X
        X = next_X / norm(next_X)
    end
    # lambda = (X^* ⋅ A ⋅ X) / (X^* ⋅ X)
    lambda = (transpose(conj(X)) * (A * X)) / (transpose(conj(X)) * X)
    lambda, X
end

poweriteration(A)

D_A, V_A = eig(A)

(D_A[1], -1 * V_A[:,1])

poweriteration

println(poweriteration(A, 1))
println(poweriteration(A, 2))
println(poweriteration(A, 5))
println(poweriteration(A, 10))

"Inner product of two vectors v, w. Can be vertical (n,1) or horizontal (1,n) vectors."
function inner(v, w)
    assert(size(v) == size(w))
    nm = size(v)
    if length(nm) == 1
        return transpose(conj(v)) * w
    elseif nm[1] == 1
        return conj(v) * transpose(w)
    end
end

inner([1 1 1], [2 3 4])

inner([1; 1; 1], [2; 3; 4])

"projection(a, e): projection operator of vector a onto base vector e. Uses inner(e, a) and inner(e, e)."
function projection(a, e)
    return (inner(e, a) / inner(e, e)) * e
end

"gramschmidt(A): Gram-Schmidt orthogonalization operator, returns us, es (unormalized matrix, base matrix)."
function gramschmidt(A, verbose=false)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    if verbose
        println("n = ", n)
    end
    us = zeros((n, n))
    es = zeros((n, n))
    for i = 1:n
        if verbose
            println("i = ", i)
        end
        us[:,i] = A[:,i]
        for j = 1:i-1
            if verbose
                println("\tj = ", j)
                println("\tus[:,i] = ", us[:,i])
            end
            us[:,i] -= projection(A[:,i], us[:,j])
        end
        if verbose
            println("us[:,i] = ", us[:,i])
        end
        es[:,i] = us[:,i] / norm(us[:,i])
        if verbose
            println("es[:,i] = ", es[:,i])
        end
    end
    return us, es
end

A

us, es = gramschmidt(A)

us

es

inner(es[:,1], A[:,1]) * es[:,1]

inner(es[:,1], A[:,2]) * es[:,1] + inner(es[:,2], A[:,2]) * es[:,2]

rank(A)

inner(es[:,1], A[:,3]) * es[:,1] + inner(es[:,2], A[:,3]) * es[:,2] + inner(es[:,3], A[:,3]) * es[:,3]

[ norm(es[:,i]) for i = 1:size(es)[1] ]

(es' * es)[1:2,1:2]

us, es = gramschmidt(B)

[ norm(es[:,i]) for i = 1:size(es)[1] ]

es * es'

V = [3 2; 1 2]

us, es = gramschmidt(V)

es' * es

"QR decomposition, returns (Q, R): Q is orthogonal and R is upper-triangular, such that A = Q R."
function QR(A)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    us, es = gramschmidt(A)
    Q = copy(es)
    R = zeros((n, n))
    for i = 1:n
        for j = i:n
            R[i,j] = inner(es[:,i], A[:,j])
        end
    end
    return Q, R
end

Q, R = QR(A)

A

Q

R

Q * R

Q2, R2 = QR(B)

B

Q2

R2

Q2 * R2

B == Q2 * R2

assert(norm(B - (Q2 * R2)) <= 1e-13)

"Apply the QR method for maxiter steps. Should return a triangular matrix similar to A (same eigenvalues)."
function QR_method(A, maxiter=50)
    Ak = A
    for k = 1:maxiter
        Qk, Rk = QR(Ak)
        Ak = Rk * Qk
    end
    return Ak
end

Ak = QR_method(A)

"Truncate to zero values under the diagonal (have to be smaller than tolerance)"
function truncate_to_zero_below_diag(A, tolerance=1e-12)
    assert(size(A)[1] == size(A)[2])
    n = size(A)[1]
    for j = 1:n
        for i = j+1:n
            assert(norm(A[i,j]) <= tolerance)
            A[i,j] = 0
        end
    end
    return A
end

truncate_to_zero_below_diag(Ak)

Ak = QR_method(B)

truncate_to_zero_below_diag(Ak)

"Extract the diagonal from the QR method."
function QR_eigenvalues(A, maxiter=50)
    return diag(QR_method(A, maxiter))
end

QR_eigenvalues(A)

eigvals(A)

QR_eigenvalues(B)

eigvals(B)

"Inverse iteration method to find the eigen vector corresponding to a given eigen value."
function inverseIteration(A, val, maxiter=20, userandomstart=true)
    mu_I = val * eye(A)
    inv_A_minus_muI = inv(A - mu_I)
    n = size(A)[1]
    if userandomstart
        X = randn((n, 1))
    else
        X = ones((n, 1))
    end
    for i = 1:maxiter
        next_X = inv_A_minus_muI * X
        X = next_X / norm(next_X)
    end
    X
end

"Approximation of D, V: D contains the eigen values (vector) and V the eigen vectors (column based)."
function QR_eigen(A, maxiter=20, userandomstart=true)
    n = size(A)[1]
    D = QR_eigenvalues(A, maxiter)
    V = zeros((n,n))
    for i = 1:n
        V[:,i] = inverseIteration(A, D[i], maxiter, userandomstart)
    end
    return D, V
end

D2, V2 = QR_eigen(B)

D2s, V2s = eig(B)


using NumericalMethodsforEngineers

a = [1. 2. -2.; 2. 5. -4.; -2. -4. 5.]

println("\n|a[1, 1]| = $(det(a[1, 1]))\n")
println("|a[1:2, 1:2]| = $(det(a[1:2, 1:2])))\n")
println("|a| = $(det(a))")
println("\nchol(a):")
chol(a) |> display

a = [16. 4. 8.; 4. 5. -4.; 8. -4. 22.]

println("\n|a[1, 1]| = $(det(a[1, 1]))\n")
println("|a[1:2, 1:2]| = $(det(a[1:2, 1:2])))\n")
println("|a| = $(det(a))")

println("\nchol(a) - upper:")
upper = chol(a)
upper |> display

println("\nchol(a) - lower:")
lower = chol(a)'
lower |> display

println("\nlower * upper:")
lower * upper |> display

@test a == lower * upper


module RandomTestMatrices

export randsym, randantisym, randposdef, randstable, randnormal, randorth, randunitary

#=

Random test matrices. These are for testing and debugging purposes, for random matrix ensembles with known limiting distributions, see the package RandomMatrices.jl .

=#

"""
randsym(d)

	Gaussian random symmetric matrix of dimension ``d``.
"""
function randsym(d)
    y = randn(d, d)
	triu(y) + triu(y,1)'
end

"""
randantisym(d)

    Anti symmetric matrix
"""
function randantisym(d)
	y = randn(d, d)
	triu(y,1) - triu(y,1)'
end

"""
randposdef(d, n = d)

	Random positive definite matrix of dimension ``d`` from the Laguerre ensemble ``n``.
"""
function randposdef(d,n = d)
	x = randn(d, n)
	x'* x/n
end

"""
randstable(d)

	Random stable matrix (matrix with eigenvalues with negative real part) with dimension ``d``.
"""
function randstable(d)
	# positive definite matrix
	x = randn(d, d)
	a = x'*x/d

	# antisymmetric matrix
	b = randantisym(d)

	# return stable matrix
	b - a
end

"""
randunitary(d)

	Random unitary matrix of dimension ``d``.
"""
randunitary(d) = expm(im* randposdef(d))

"""
randorth(d)

	Orthogonal matrix drawn according to the Haar measure on the group of orthogonal matrices.
"""
randorth(d) = qr(randn(d,d))[1]

function tridiag(di, u, l)
	assert(length(di) -1 == length(u) == length(l))
	M = diagm(di)
	for i in 1:length(u)
	  M[i,i+1] = l[i]
  	  M[i+1,i] = u[i]
	end
	M

end

"""
randnormal(d)

	Random normal matrix of dimension ``d``.
"""
function randnormal(d)
	Q = randorth(d)
	B = schur(randn(d,d))[1] #same eigenvalues as a randn(d,d)
	m = d
	alpha = zeros(d)
	beta = zeros(d-1)
	while m > 1
	   s = abs(B[m-1,m-1]) + abs(B[m,m])
	   if s + abs(B[m,m-1]) > s # if significant offdiagonal value: evaluate submatrix m-1:m;
		spur = B[m,m] + B[m-1,m-1]
		dis2 = abs2(B[m,m]- B[m-1,m-1]) + 4.0*B[m,m-1]*B[m-1,m] # = spur^2 - 4det
		alpha[m] = alpha[m-1] = 0.5*spur

		beta[m-1] = 0.5*sqrt(-dis2)
		m -= 2
	   else
	        alpha[m] = B[m,m]
	        beta[m-1] = 0
	        m -= 1
	   end

	end
	if (m == 1)
		alpha[1] = B[1,1]
	end

	Q' * tridiag(alpha, beta, -beta) * Q

end

end # module


# # Factorizations and other fun
# Based on work by Andreas Noack
#
# ## Outline
#  - Factorizations
#  - Special matrix structures
#  - Generic linear algebra

#-

# Before we get started, let's set up a linear system and use `LinearAlgebra` to bring in the factorizations and special matrix structures.

using LinearAlgebra
A = rand(3, 3)
x = fill(1, (3,))
b = A * x

# ## Factorizations
#
# #### LU factorizations
# In Julia we can perform an LU factorization
# ```julia
# PA = LU
# ```
# where `P` is a permutation matrix, `L` is lower triangular unit diagonal and `U` is upper triangular, using `lufact`.
#
# Julia allows computing the LU factorization and defines a composite factorization type for storing it.

Alu = lu(A)

#-

typeof(Alu)

# The different parts of the factorization can be extracted by accessing their special properties

Alu.P

#-

Alu.L

#-

Alu.U

# Julia can dispatch methods on factorization objects.
#
# For example, we can solve the linear system using either the original matrix or the factorization object.

A\b

#-

Alu\b

# Similarly, we can calculate the determinant of `A` using either `A` or the factorization object

det(A) ≈ det(Alu)

# #### QR factorizations
#
# In Julia we can perform a QR factorization
# ```
# A=QR
# ```
#
# where `Q` is unitary/orthogonal and `R` is upper triangular, using `qrfact`.

Aqr = qr(A)

# Similarly to the LU factorization, the matrices `Q` and `R` can be extracted from the QR factorization object via

Aqr.Q

#-

Aqr.R

# #### Eigendecompositions

#-

# The results from eigendecompositions, singular value decompositions, Hessenberg factorizations, and Schur decompositions are all stored in `Factorization` types.
#
# The eigendecomposition can be computed

Asym = A + A'
AsymEig = eigen(Asym)

# The values and the vectors can be extracted from the Eigen type by special indexing

AsymEig.values

#-

AsymEig.vectors

# Once again, when the factorization is stored in a type, we can dispatch on it and write specialized methods that exploit the properties of the factorization, e.g. that $A^{-1}=(V\Lambda V^{-1})^{-1}=V\Lambda^{-1}V^{-1}$.

inv(AsymEig)*Asym

# ## Special matrix structures
# Matrix structure is very important in linear algebra. To see *how* important it is, let's work with a larger linear system

n = 1000
A = randn(n,n);

# Julia can often infer special matrix structure

Asym = A + A'
issymmetric(Asym)

# but sometimes floating point error might get in the way.

Asym_noisy = copy(Asym)
Asym_noisy[1,2] += 5eps()

#-

issymmetric(Asym_noisy)

# Luckily we can declare structure explicitly with, for example, `Diagonal`, `Triangular`, `Symmetric`, `Hermitian`, `Tridiagonal` and `SymTridiagonal`.

Asym_explicit = Symmetric(Asym_noisy);

# Let's compare how long it takes Julia to compute the eigenvalues of `Asym`, `Asym_noisy`, and `Asym_explicit`

@time eigvals(Asym);

#-

@time eigvals(Asym_noisy);

#-

@time eigvals(Asym_explicit);

# In this example, using `Symmetric()` on `Asym_noisy` made our calculations about `5x` more efficient :)

#-

# #### A big problem
# Using the `Tridiagonal` and `SymTridiagonal` types to store tridiagonal matrices makes it possible to work with potentially very large tridiagonal problems. The following problem would not be possible to solve on a laptop if the matrix had to be stored as a (dense) `Matrix` type.

n = 1_000_000;
A = SymTridiagonal(randn(n), randn(n-1));
@time eigmax(A)

# ## Generic linear algebra
# The usual way of adding support for numerical linear algebra is by wrapping BLAS and LAPACK subroutines. For matrices with elements of `Float32`, `Float64`, `Complex{Float32}` or `Complex{Float64}` this is also what Julia does.
#
# However, Julia also supports generic linear algebra, allowing you to, for example, work with matrices and vectors of rational numbers.

#-

# #### Rational numbers
# Julia has rational numbers built in. To construct a rational number, use double forward slashes:

1//2

# #### Example: Rational linear system of equations
# The following example shows how linear system of equations with rational elements can be solved without promoting to floating point element types. Overflow can easily become a problem when working with rational numbers so we use `BigInt`s.

Arational = Matrix{Rational{BigInt}}(rand(1:10, 3, 3))/10

#-

x = fill(1, 3)
b = Arational*x

#-

Arational\b

#-

lu(Arational)

# ### Exercises
#
# #### 11.1
# What are the eigenvalues of matrix A?
#
# ```
# A =
# [
#  140   97   74  168  131
#   97  106   89  131   36
#   74   89  152  144   71
#  168  131  144   54  142
#  131   36   71  142   36
# ]
# ```
# and assign it a variable `A_eigv`

using LinearAlgebra

#-

@assert A_eigv ==  [-128.49322764802145, -55.887784553056875, 42.7521672793189, 87.16111477514521, 542.4677301466143]

# #### 11.2
# Create a `Diagonal` matrix from the eigenvalues of `A`.

@assert A_diag ==  [-128.493    0.0      0.0      0.0       0.0;
    0.0    -55.8878   0.0      0.0       0.0;
    0.0      0.0     42.7522   0.0       0.0;
    0.0      0.0      0.0     87.1611    0.0;
    0.0 0.0      0.0      0.0     542.468]
## #### 11.3
## Create a `LowerTriangular` matrix from `A` and store it in `A_lowertri`
@assert A_lowertri ==  [140    0    0    0   0;
  97  106    0    0   0;
  74   89  152    0   0;
 168  131  144   54   0;
 131   36   71  142  36]


# Mld.jl

###### mld #####
"""
    mld(v)

Compute the Mean log deviation of a vector `v`.

# Examples
```julia
julia> using Inequality
julia> mld([8, 5, 1, 3, 5, 6, 7, 6, 3])
0.1397460530936332
```
"""
mld(v::AbstractVector{<:Real})::Float64 = log(Statistics.mean(v[v .!= 0])) - Statistics.mean(log.(v[v .!= 0]))

###### weighted mld #####
"""
    mld(v, w)

Compute the weighted Mean log deviation of a vector `v` using weights given by a weight vector `w`.

Weights must not be negative, missing or NaN. The weights and data vectors must have the same length.

# Examples
```julia
julia> using Inequality
julia> mld([8, 5, 1, 3, 5, 6, 7, 6, 3], collect(0.1:0.1:0.9))
0.10375545537468206
```
"""
function mld(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64
    checks_weights(v, w)
    w = w[v .!= 0]
    v = v[v .!= 0]

    return -sum(w .* log.(v/mean(v, weights(w))) ) / sum(w)
end

function mld(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64
    checks_weights(v, w)
    w = w[v .!= 0]
    v = v[v .!= 0]

    return -sum(w .* log.(v/mean(v, weights(w))))/ sum(w)
end

"""
    wmld(v, w)

Compute the Mean log deviationof `v` with weights `w`. See also [`mld`](@mld)
"""
wmld(v::AbstractVector{<:Real}, w::AbstractVector{<:Real}) = mld(v, w)

wmld(v::AbstractVector{<:Real}, w::AbstractWeights) = mld(v, w)


struct ODEProblem{F,T<:Tuple{Number,Number},U<:AbstractVector,P<:AbstractVector}
    f::F
    tspan::T
    u0::U
    θ::P
end

abstract type ODESolver end

struct Euler{T} <: ODESolver
    dt::T
end

function (solver::Euler)(prob::ODEProblem, u, t)
    f, θ, dt  = prob.f, prob.θ, solver.dt
    (u + dt*f(u,θ), t+dt)
end

function solve(prob::ODEProblem, solver::ODESolver)
    t = prob.tspan[1]; u = prob.u0
    us = [u]; ts = [t]
    while t < prob.tspan[2]
        (u,t) = solver(prob, u, t)
        push!(us,u)
        push!(ts,t)
    end
    ts, reduce(hcat,us)
end

# Define & Solve ODE

function lotkavolterra(x,θ)
    α, β, γ, δ = θ
    x₁, x₂ = x

    dx₁ = α*x₁ - β*x₁*x₂
    dx₂ = δ*x₁*x₂ - γ*x₂

    [dx₁, dx₂]
end

θ = [0.1,0.2,0.3,0.2]
u0 = [1.0,1.0]
tspan = (0.,100.)
prob = ODEProblem(lotkavolterra,tspan,u0,θ)