#############################
##  These numerical experiments, included for the sake of completion,
##  showcase some numerical solutions to LINEAR Volterra integral equations
#############################

using ApproxFun, MultivariateOrthogonalPolynomials, BandedMatrices, BlockBandedMatrices, SpecialFunctions, Plots
using SparseVolterraExamples

#####################################################
## Solving Equation 18 in https://doi.org/10.1137/19M1267441
## This is a linear Volterra integral equation of FIRST kind.
#####################################################
####
## Set up g(x) and kernel K(x,y)
g1(x)= exp(-x)/4+1/4*exp(x)*(-1+2*x)
K(x,y) = exp(y-x)
####
## For linear problems the solver is wrapped into convenient functions. N is polynomial order of approximation.
N = 20
u = triVolterraEQ1FullKernelSolver(g1,K,N,true)
####
## Plot the computed and analytic solution
plot(u,grid=false,ylabel="u(x)",xlabel="x",label="sparse method")
plot!(x->x*exp(x),0,1,grid=false,ylabel="u(x)",xlabel="x",label="analytic")
####
## Plot error
plot(triVolterraEQ1FullKernelSolver(g1,K,N,true)-Fun(x->x*exp(x),Jacobi(0,1,0..1)),ylabel="error",xlabel="x",grid=false,label=false)

#####################################################
## The following is a simple linear second kind Volterra integral equation, not appearing in either paper.
## It uses the monomial coefficient vector input kernel implementation and limits of integration to 1-x instead and has analytically known solution sin(-2*pi*x).
#####################################################
####
## Set up g(x) and kernel K(x,y)
g2 = Fun(x->(x-1)/(2*pi)*cos(-2*pi*x)+(4*pi^2+1)/(4*pi^2)*sin(2*pi*(1-x)), Jacobi(0,1,0..1))
K2 = [0.,1.,0.,0.] # this is equivalent to K(x,y)=y
####
## Compute and plot solutions
N = 20
u = triVolterraEQ2Solver(g2,K2,N,false) # note the false indicating that we are not flipping the integration bounds, i.e. we are integrating from 0 to 1-x
plot(u,grid=false,ylabel="u(x)",xlabel="x",label="sparse method")
plot!(x->sin(-2*pi*x),0,1,grid=false,ylabel="u(x)",xlabel="x",label="analytic")
####
## Plot errors
plot(triVolterraEQ2Solver(g2,K2,N,false)-Fun(x->sin(-2*pi*x),Jacobi(0,1,0..1)),grid=false,ylabel="error",xlabel="x",label=false)
####
## The above is equivalent to the following much simpler input, which uses the full kernel input as a function
u2 = triVolterraEQ2FullKernelSolver(g2,(x,y)->y,N,false)
####
## Verify the two input methods give the same answer
plot(x->abs(u2(x)-u(x)),0,1,grid=false,ylabel="difference",xlabel="x",label=false)

examples/Example 0 - Some linear Volterra integral equation examples.jl


begin
  import Pkg; Pkg.activate(".")
  Pkg.add("DifferentialEquations")
  Pkg.add("Optim")
end

using DifferentialEquations, Plots

function f(du,u,p,t)
  du[1] = u[2]
  du[2] = -p
end

u₀ = [50.0,0.0]
tspan = (0.0,15.0)
p = 9.8

prob = ODEProblem(f,u₀,tspan, p)
sol = solve(prob,Tsit5())
plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

# But where is the ground?
function condition(u,t,integrator) # Event when event_f(u,t) == 0
  u[1]
end

function affect!(integrator)
  integrator.u[2] = -integrator.u[2]
end

cb = ContinuousCallback(condition,affect!, save_positions=(false, false))

sol = solve(prob,Tsit5(), callback=cb)
plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

### Air resistance
function f₂(du,u,p,t)
  g, Cd = p
  du[1] = u[2]
  du[2] = -(g+u[2]^2*Cd*sign(u[2]))
end

p = (9.81, 15e-2)

prob = ODEProblem(f₂, u₀,tspan, p)
sol = solve(prob, Tsit5(), callback=cb)

plot(sol, label = ["Displacement" "Velocity"],
  xlabel="Time (s)",
  ylabel="Displacement (m) | Velocity (m/s)")

### Optim.jl
using Optim

function compute_ODE(p; times=1:0.5:15)
  prob = ODEProblem(f₂, u₀,tspan, p)
  sol = solve(prob, Tsit5(), saveat=times, callback = cb)
  return sol.t , hcat(sol.u...)[1,:]
end

time, exp_disp = compute_ODE((9.81, 5e-2))
exp_disp .+= 0.75randn(length(exp_disp)) # add noise

scatter(time, exp_disp, label="Experiment", xlabel="Time(s)", ylabel="Displacement (m)")

function objective(Cd)
  _, sim_disp = compute_ODE((9.81, Cd[1]))
  return sum((sim_disp.- exp_disp).^2)
end

res = optimize(objective, [5e-3], Newton(), Optim.Options(store_trace=true, extended_trace=true))
Cd = res.minimizer[1]

plot!(compute_ODE((9.81, Cd); times=0:0.1:15), label="Simulated")

### Visual
for iteration in res.trace[1:10]
  Cd = iteration.metadata["x"][1]
  pl = scatter(time, exp_disp, label="Experiment", xlabel="Time(s)", ylabel="Displacement (m)")
  plot!(pl, compute_ODE((9.81, Cd); times=0:0.1:15), label="Simulated (Cd: $Cd")
  display(pl)
  sleep(1.0)
end


x = rand(1:100, 5)
println("Sum of ", x, " = ", sum(x))

A = rand(-100:100, 3, 5)

sum(A, dims=1)

sum(A, dims=2)

sum(A, dims=1)[:]

x = rand(1:100, 5)
display(x)
prod(x)                         # Product of all elements

display(A)
display(maximum(A, dims=1))     # Largest element in each column
display(minimum(A, dims=1))     # Smallest element in each column

x = 1:5
display(cumsum(x))                   # Cumulative sum, that is, entry n is the sum of x_1,...,x_n
display(cumprod(x))                  # Cumulative product

A = reshape(1:6, 2, 3)

cumsum(A, dims=1)                    # Cumulative sum along dimension 1 (that is, column-wise)

cumprod(A, dims=2)                   # Cumulative product along dimension 2

function taylor_cos(x,n)
    factors = [ -x^2 / ((2k-1) * 2k) for k = 1:n ]
    y = 1 + sum(cumprod(factors))
end

println(taylor_cos(10, 50)) # Taylor approximation
println(cos(10))            # true value

xx = -10:0.01:10
yy = cos.(xx)
yy_taylor = [ taylor_cos(x,n) for x in xx, n in 0:5 ]

using PyPlot
plot(xx, yy, linewidth=2, color="k")
plot(xx, yy_taylor)
grid(true)
axis([-10,10,-1.2,1.2]);

x = 1:2:1000                 # Odd numbers
println(503 in x)            # True, 503 is in the list
println(1000 ∈ x)            # False, 1000 is not in the list

println(all(x .< 500))
println(all(x .> 0))

println(any(x .== 503))
println(any(x .== 1000))

idx = findfirst(x .== 503)    # Index in x with the value 503

ind = findall(@. x % 97 == 0)
println("Indices in x with numbers that are multiples of 97: ", ind)

count(@. x % 97 == 0)


# Load X and y variable
using JLD
data = load("logisticData.jld")
(X,y) = (data["X"],data["y"])

# Add bias variable, initialize w, set regularization and optimization parameters
(n,d) = size(X)
X = [ones(n,1) X]
d += 1
w = zeros(d,1)
lambda = 1
maxPasses = 500
progTol = 1e-4
verbose = true

## Run and time coordinate descent to minimize L2-regularization logistic loss

# Start timer
tic()

# Compute Lipschitz constant of 'f'
(eigenvalues,~) = eig(X'*X)
L = .25*maximum(eigenvalues) + lambda;

# Start running coordinate descent
w_old = copy(w);
for k in 1:maxPasses*d

    # Choose variable to update 'j'
    j = rand(1:d)

    # Compute partial derivative 'g_j'
    yXw = y.*(X*w);
    sigmoid = 1./(1+exp.(-yXw));
    g = -X'*(y.*(1-sigmoid)) + lambda*w;
    g_j = g[j];

    # Update variable
    w[j] -= (1/L)*g_j;

    # Check for lack of progress after each "pass"
    if mod(k,d) == 0
        yXw = y.*(X*w)
        f = sum(log.(1 + exp.(-y.*(X*w)))) + (lambda/2)norm(w)^2
        delta = norm(w-w_old,Inf);
        if verbose
            @printf("Passes = %d, function = %.4e, change = %.4f\n",k/d,f,delta);
        end
        if delta < progTol
            @printf("Parameters changed by less than progTol on pass\n");
            break;
        end
        w_old = copy(w);
    end

end

# End timer
toc()


A = rand(-4:4,5,3)
B = rand(-4:4,3,4)

C = A*B

C[2,3]

m = 5; n = 3; p = 4;
C = zeros(m,p);
for i = 1:m
    for j = 1:p
        for k = 1:n
            C[i,j] += A[i,k]*B[k,j]
        end
    end
end

C

m = 5; n = 3; p = 4;
C = zeros(m,p);
for i = 1:m
    for j = 1:p
        C[i,j] = sum( A[i,k]*B[k,j] for k = 1:n );
    end
end
C

m = 5; n = 3; p = 4;
C = [ sum( A[i,k]*B[k,j] for k = 1:n ) for i=1:m, j=1:p ]

# C is a matrix of inner products
i=2; j=3;
using LinearAlgebra
dot(A[i,:],B[:,j])
C = [ dot(A[i,:],B[:,j]) for i=1:m, j=1:p ]

# C columnwise is a linear combination of columns
C[:,j]
sum( B[k,j]*A[:,k] for k=1:n  )
hcat( [ sum( B[k,j]*A[:,k] for k=1:n  ) for j=1:p ]... )
q = [ sum( B[k,j]*A[:,k] for k=1:n  ) for j=1:p ];
hcat(q[1],q[2],q[3],q[4])

length(C)

length(q)

# C is a sum of outer products
sum( A[:,k]*B[k,:]' for k=1:n)

B[1,:]'

(B[1,:]')'


struct LineSearchOptions
    sufficient_decrease::Float64
    curvature::Float64
    max_iter::Int
end

function LineSearchOptions(; sufficient_decrease = 1e-4, curvature = 0.1, max_iter = 100)
    LineSearchOptions(sufficient_decrease, curvature, max_iter)
end

function line_search(
    fcn,
    x0,
    step_direction,
    f0 = fcn(x0),
    search_options::LineSearchOptions = LineSearchOptions())
# suff_decrease = 1e-4, curvature = 0.1, max_iter = 100)

    # rename for ease of refactoring
    suff_decrease = search_options.sufficient_decrease
    curvature     = search_options.curvature
    max_iter      = search_options.max_iter

    # strong wolfe condition
    step_size_prev = zero(eltype(x0))
    step_size      = one(eltype(x0))
    step_size_lo   = step_size_prev
    step_size_hi   = step_size_prev
    f_prev         = NaN
    dphi0          = numder.directional_diff_fw(fcn, x0, step_direction)

    # bracket phase
    for iter = 1:max_iter

        f1 = fcn(x0 + step_direction * step_size)
        if f1 > f0 + suff_decrease * step_size * dphi0 || (iter > 1 && f1 >= f_prev)
            step_size_lo, step_size_hi = step_size_prev, step_size
            break
        end

        dphi = numder.directional_diff_fw(fcn, x0 + step_direction * step_size, step_direction)
        if abs(dphi) <= abs(dphi0) * curvature
            return step_size
        elseif dphi >= 0
            step_size_lo, step_size_hi = step_size, step_size_prev
            break
        end

        f_prev = f1
        step_size_prev, step_size = step_size, step_size * 2
    end

    # zoom phase
    f_lo = fcn(x0 + step_size_lo * step_direction)
    for iter = 1:max_iter
        step_size = 0.5 * (step_size_lo + step_size_hi)
        f1 = fcn(x0 + step_size * step_direction)
        if f1 > f0 + suff_decrease * step_size * dphi0 || f1 >= f_lo
            step_size_hi = step_size
        else
            dphi = numder.directional_diff_fw(fcn, x0 + step_direction * step_size, step_direction)
            if abs(dphi) <= abs(dphi0) * curvature
                return step_size
            elseif dphi * (step_size_hi - step_size_lo) >= 0
                step_size_hi = step_size_lo
            end
            step_size_lo = step_size
        end

    end

    return step_size
end

#=
function armijo_backtracking(fcn, x0, direction, f0 = fcn(x0), alpha_0 = 1, step_factor = 0.5, max_iteration = 20, beta = 1e-4)
    # sufficient decrease condition shall be satisfied
    phi_0 = f0
    dphi_dalpha_0 = NumDiff.directional_diff_fw(fcn, x0, direction)

    alpha = alpha_0
    for iter = 1:max_iteration
        phi_alpha = fcn( x0 + direction .* alpha)
        if phi_alpha <= phi_0 + beta * alpha * phi_alpha
            break
        end
        alpha = alpha * step_factor
    end
end
=#


## Exercise 7-2
## Copy the loop from Square Roots and encapsulate it in a function called mysqrt that takes a as a parameter, chooses a reasonable value of x, and returns an estimate of the square root of a.

function mysqrt(a, x)
    a = float(a)
    while true
        y = (x + a/x) / 2
        if y == x
            break
        end
        x = y
    end
    return x
end

## To test it, write a function named testsquareroot that prints a table like this:
# see ex 7-2 in book
println("Ans: ")

function rightjustify(string::String, chr)
    if chr - length(string) > 0
        return ' ' ^ (chr - length(string)) * string
    end
    return string
end

function testsquareroot()
    println("a   mysqrt             sqrt               diff")
    println("-   ------             ----               ----")
    for i in 1:10
        i = float(i)
        root = sqrt(i)
        root_my = mysqrt(i, i/2)
        println(
            string(i) * " " *
            rightjustify(string(root_my), 18) *
            " " *
            rightjustify(string(root), 18) * " " * string(abs(root_my - root)))
    end
end

testsquareroot()

println("End.")


# hann.jl
# Hann window function

"""
    hann(x::AbstractFloat)
Hann window function (often called "Hanning" window function).
This version is `0.5 * (1 + cospi(2x))` and used in `[-0.5,+0.5]`.
https://en.wikipedia.org/wiki/Hann_function
"""
hann(x::AbstractFloat) = 0.5 * (1 + cospi(2x))

"""
    hann(n::Int)
Return vector of `n ≥ 2` samples of Hann window function
equally spaced over `[-0.5,+0.5]`.
The first and last samples are zero for `n ≥ 2`.

Caution.
Matlab has both `hann` and `hanning` functions for which
`[0; hanning(n); 0] == hann(n+2)`.

`DSP.hanning(n)` == Matlab.hann(n)`
whereas
`Sound.hann(n) == Matlab.hanning(n)`.
"""
hann(n::Int) = hann.(LinRange(-0.5, 0.5, n+2))[2:end-1]


using Random, Distributions, KernelDensity, Plots, LaTeXStrings; pyplot()
Random.seed!(1)

function tStat(mu0,mu,sig,n)
    sample = rand(Normal(mu,sig),n)
    xBar   = mean(sample)
    s      = std(sample)
    (xBar-mu0)/(s/sqrt(n))
end

mu0, mu1A, mu1B = 20, 22, 24
sig, n = 7, 5
N = 10^6
alpha = 0.05

dataH0  = [tStat(mu0,mu0,sig,n) for _ in 1:N]
dataH1A = [tStat(mu0,mu1A,sig,n) for _ in 1:N]
dataH1B = [tStat(mu0,mu1B,sig,n) for _ in 1:N]
dataH1C = [tStat(mu0,mu1B,sig,2*n) for _ in 1:N]
dataH1D = [tStat(mu0,mu1B,sig/2,2*n) for _ in 1:N]

tCrit = quantile(TDist(n-1),1-alpha)
estPwr(sample) = sum(sample .> tCrit)/N

println("Rejection boundary: ", tCrit)
println("Power under H0:  ", estPwr(dataH0))
println("Power under H1A: ", estPwr(dataH1A))
println("Power under H1B (mu's farther apart): ", estPwr(dataH1B))
println("Power under H1C (double sample size): ", estPwr(dataH1C))
println("Power under H1D (like H1C but std/2): ", estPwr(dataH1D))

kH0  = kde(dataH0)
kH1A = kde(dataH1A)
kH1D = kde(dataH1D)
xGrid = -10:0.1:15

plot(xGrid,pdf(kH0,xGrid),
	c=:blue, label="Distribution under H0")
plot!(xGrid,pdf(kH1A,xGrid),
	c=:red, label="Distribution under H1A")
plot!(xGrid,pdf(kH1D,xGrid),
	c=:green, label="Distribution under H1D")
plot!([tCrit,tCrit],[0,0.4],
	c=:black, ls=:dash, label="Critical value boundary",
	xlims=(-5,10), ylims=(0,0.4), xlabel=L"\Delta = \mu - \mu_0")


using SpecialFunctions, Distributions, Roots, Plots, LaTeXStrings; pyplot()

eq(alpha, xb, xbl) = log(alpha) - digamma(alpha) - log(xb) + xbl

actualAlpha, actualLambda = 2, 3
gammaDist = Gamma(actualAlpha,1/actualLambda)

function mle(sample)
    alpha  = find_zero( (a)->eq(a,mean(sample),mean(log.(sample))), 1)
    lambda = alpha/mean(sample)
    return [alpha,lambda]
end

N = 10^4

mles10   = [mle(rand(gammaDist,10)) for _ in 1:N]
mles100  = [mle(rand(gammaDist,100)) for _ in 1:N]
mles1000 = [mle(rand(gammaDist,1000)) for _ in 1:N]

scatter(first.(mles10), last.(mles10),
	c=:blue, ms=1, msw=0, label="n = 10")
scatter!(first.(mles100), last.(mles100),
	c=:red, ms=1, msw=0, label="n = 100")
scatter!(first.(mles1000), last.(mles1000),
	c=:green, ms=1, msw=0, label="n = 1000",
	xlims=(0,6), ylims=(0,8), xlabel=L"\alpha", ylabel=L"\lambda")


export dde_stability

"""
    dde_stability(A, B, τ; N=10)

Calculate (a finite number of) the characteristic roots of the linear delay differential
equation

```math
x'(t) = Ax(t) + Bx(t-τ)
```

where `A` and `B` can be (square) matrices. `N` controls the order of the Chebyshev
polynomial approximation to the infinitesimal generator; higher values of `N` result in a
more accurate approximation. A property of the algorithm used is that characteristic roots
closest to the origin are approximated best.

## Reference

Pseudospectral Differencing Methods for Characteristic Roots of Delay Differential
Equations; D. Breda, S. Maset, and R. Vermiglio, SIAM Journal on Scientific Computing 2005
27:2, 482-495
"""
function dde_stability(A, B, τ; N=10)
    @assert size(A) == size(B)
    @assert size(A, 1) == size(A, 2)
    T = eltype(A) == Int ? Float64 : eltype(A)
    n = size(A, 1)
    (x, D) = cheb_diff(T, N)
    M = kron(-D, Matrix(I / convert(T, τ), n, n))
    M[1:n, 1:n] .= A
    M[1:n, (n + 1):(end - n)] .= 0
    M[1:n, (end - n + 1):end] .= B
    return eigen(M)
end

# Tests
# 1. x'(t) = 2x(t) - exp(1)*x(t - 1) has a double root at λ = 1
# 2. x'(t) = -x(t - τ) goes unstable at τ=π/2
# 3. x'(t) = α*A*x(t) + (1-α)*A*x(t-τ)
#    A = [
#          50  284   41   23   50   32
#        -280  -46  -19  -37  -10  -28
#          35   -1   26  143   35   17
#          5   -31 -139  -22    5  -13
#          20  -16   11   -7 -115  137
#         -10  -46  -19  -37 -145 -163
#    ]
#    with α=0.9 is stable for 3π/(270*(2*0.9 - 1)) ≤ τ ≤ 4π/270
#    that is 0.04363323129985824 ≤ τ ≤ 0.046542113386515455


# Inverse and soln of matrix
# Gaussian
# generally JL handles eqns and invs

# we use Finverse to denote function to find inverse
# 🙈

function Finverse_Gaussian(x)
    n = Int(sqrt(length(x)))
    x = reshape(x,n,n)
    I = zeros(n,n)
    for i in 1:n
        I[i,i] =1
    end
    for i in 1:(n-1)
        for j in (i+1):n
            m = x[j,i]/x[i,i]
            x[j,i:end] = x[j,i:end].-x[i,i:end].*m
            I[j,:] = I[j,:].-I[i,:].*m
        end
    end
    for i in n:-1:2
        for j in (i-1):-1:1
            m = x[j,i]/x[i,i]
            x[j,i]=0
            I[j,:] =I[j,:] .- I[i,:].*m
        end
    end
    for i in 1:n
        I[i,:] = I[i,:]./x[i,i]
    end
    return I
end

function Fsolve_Gaussian(x,b)
    n=length(b)
    x = reshape(x,n,n)
    for i in 1:(n-1)
        for j in (i+1):n
            m = x[j,i]/x[i,i]
            x[j,i:end] = x[j,i:end].-x[i,i:end].*m
            b[j] = b[j]-b[i]*m
        end
    end
    return b
end

Fsolve_Gaussian([1 2 3; 4 5 6;7 8 9],[1;2;3])
[1 2 3; 4 5 6;7 8 9]*[1 ,-2, 0]

#Finverse_Gaussian([1 2;6 7])

#[1 2;6 7]*Finverse_Gaussian([1 2;6 7])

# safer version of Gaussian solve -judge and shift terms with 0

#function Fsolve_Gaussian_safely(x,b)
#    n=length(b)
#    for j in 1:(n-1)
#    end
#end

x = [1 2 3;4 5 6;7 8 9]

function LU(x)
    n=Int(sqrt(length(x)))
    for j in 1:(n-1)
        for i in (j+1):n
            m = x[i,j] / x[j,j]
            x[i,(j+1):n] = x[i,(j+1):n].-m.*x[j,(j+1):n]
            x[i,j] = m
        end
    end
    return x
end
LU(x)


function bicgstab(a::Matrix{Float64}, b::Vector{Float64}, x0::Vector{Float64};
  tol=1.0e-10, limit=1000, trace=false)

  n = size(a, 1)
  x = copy(x0)
  local converged

  r = b - a * x
  r0_hat = copy(r)
  rho0 = w = α = 1.0
  v = zeros(n)
  p = zeros(n)
  rho1 = dot(r0_hat, r)
  trace && println("\nFirst few iterations:")
  iters = 0
  while true
    iters += 1
    converged = norm(r) < tol * norm(b)
    (converged || (iters == limit)) && break
    β = (rho1 / rho0) * (α / w)
    p = r + β * (p - w * v)
    v = a * p
    α = rho1 / dot(r0_hat, v)
    s = r - α * v
    t = a * s
    w = dot(t, s) / dot(t, t)
    rho0 = rho1
    rho1 = -w * dot(r0_hat, t)
    x += α * p + w * s
    r = s - w * t
    if trace && iters <= 5
      println("After $iters iterations: x = $x")
    end
  end
  iters, converged, x
end


#=
  Original Author: Anastasia Yendiki

  Copyright © 2022 The General Hospital Corporation (Boston, MA) "MGH"

  Terms and conditions for use, reproduction, distribution and contribution
  are found in the 'FreeSurfer Software License Agreement' contained
  in the file 'LICENSE' found in the FreeSurfer distribution, and here:

  https://surfer.nmr.mgh.harvard.edu/fswiki/FreeSurferSoftwareLicense

  Reporting: freesurfer@nmr.mgh.harvard.edu
=#

export cart2pol, pol2cart, cart2sph, sph2cart

"""
    cart2pol(x, y)

    Transform Cartesian coordinates (`x`, `y`) to polar coordinates (`φ`, `ρ`),
    where `φ` is in radians.
"""
function cart2pol(x, y)

  φ = atan.(y, x)
  ρ = hypot.(x, y)

  return φ, ρ
end

"""
    pol2cart(φ, ρ)

    Transform polar coordinates (`φ`, `ρ`) to Cartesian coordinates (`x`, `y`),
    where `φ` is in radians.
"""
function pol2cart(φ, ρ)

  x = ρ .* cos.(φ)
  y = ρ .* sin.(φ)

  return x, y
end

"""
    cart2sph(x, y, z)

    Transform Cartesian coordinates (`x`, `y`, `z`) to spherical coordinates
    (`φ`, `θ`, `ρ`), where `φ` and `θ` are in radians.
"""
function cart2sph(x, y, z)

  hypotxy = hypot.(x, y)
  ρ = hypot.(hypotxy, z)
  θ = atan.(z, hypotxy)
  φ = atan.(y, x)

  return φ, θ, ρ
end

"""
    sph2cart(φ, θ, ρ)

    Transform spherical coordinates (`φ`, `θ`, `ρ`) to Cartesian coordinates
    (`x`, `y`, `z`).

    `φ` and `θ` must be in radians.
"""
function sph2cart(φ, θ, ρ)

  z = ρ .* sin.(θ)
  ρcosθ = ρ .* cos.(θ)
  x = ρcosθ .* cos.(φ)
  y = ρcosθ .* sin.(φ)

  return x, y, z
end

"""
    ang2rot(φ, θ)

    Convert polar `φ` and azimuthal `θ` angles to a rotation matrix.

    `φ` and `θ` must be in radians.
"""
function ang2rot(φ, θ)

  # Rotate by φ around the z-axis
  Rz = [cos(φ)  -sin(φ)  0
        sin(φ)   cos(φ)  0
        0         0      1]

  # Rotate by θ around the y-axis
  Ry = [cos(θ)   0   sin(θ)
        0        1   0
       -sin(θ)   0   cos(θ)]

  R =  Rz * Ry

  return R
end


function subbac!(u::Matrix{Float64}, b::Vector{Float64})
# Back-substitution on an Upper Triangle
  n = size(u, 1)
  for i in n:-1:1
    total = b[i]
    if i < n
      for j in i+1:n
        total -= u[i, j] * b[j]
      end
    end
    b[i] = total / u[i, i]
  end
end

function subbac(u::Matrix{Float64}, b::Vector{Float64})
# Back-substitution on an Upper Triangle
  bt = deepcopy(b)
  n = size(u, 1)
  for i in n:-1:1
    total = bt[i]
    if i < n
      for j in i+1:n
        total -= u[i, j] * bt[j]
      end
    end
    bt[i] = total / u[i, i]
  end
  bt
end

export
  subbac!,
  subbac


"""
    pearson_correlation(x, y)

Find the pearson correlation between two variables.

# Example:

julia> PearsonCorrelation([12,11,16,17,19,21],[11,51,66,72,12,15])
-0.2092706263573845

Contribution by: [Aru Bhardwaj](https://github.com/arubhardwaj)

"""

function pearson_correlation(x, y)
    mean_x = sum(x) / length(x)
    mean_y = sum(y) / length(y)
    XY = (x .- mean_x) .* (y .- mean_y)
    XXs = sum((x .- mean_x) .* (x .- mean_x))
    YYs = sum((y .- mean_y) .* (y .- mean_y))
    return(sum(XY) / (sqrt(XXs .* YYs)))
end


# Monte-Carlo Applications

# Type 1
# path like integration
# ∫₀¹ e
# todo

# Type 2
# Volume-like integration

# E.g. evaluate π

# 1. generate n points within a unit □
#    enclosed by xlim<-[0 1]
#    and         ylim<-[0 1]

n = 1000
x = rand(n)
y = rand(n)

# 2. Judge # of pts in the unit ◷
s = 0
for i in 1:n# I hate for loop , there's a simpler version below
    if (x[i]^2 + y[i]^2)<=1
        global s+=1
    end
end
s

# Area of ▧ / Area of ◍  = 4/π
π_est = 4s/n
π_est

# Vectorize? replace 2 with
#even faster?
@time size(zeros(n)[x.^2+y.^2 .<=1])
#0.000070 seconds (27 allocations: 42.813 KiB)
#Compared with

s=0
@time for i in 1:n
    if (x[i]^2 + y[i]^2)<=1
        global s+=1
    end
end
#0.000220 seconds (7.75 k allocations: 136.734 KiB)

π_est = 4s/n
π_est

plot(x,y;seriestype="scatter")
plot!(0:.01:1,sqrt.(1 .-(0:.01:1).^2))


# determine the moment of the stoichiometric matrix coefficient
# necessary to deal with (independent) stochastic variables
# NOTE: only geometric distribution is supported
# the parameter passed is supposed to be the MEAN value
#function expected_coeff(x::Union{Int, Sym{ModelingToolkit.Parameter{Real}}}, pwr)
function expected_coeff(x::Union{Int, Symbolic}, pwr)
    if typeof(x) == Int64
        x^pwr
    else
        geometric_raw_moment(x, pwr)
    end
end

function eulerian_number(n, k)
    suma = 0
    for j in 0:k+1
        suma += (-1)^j * binomial(n+1, j) * (k-j+1)^n
    end
    return suma
end

#function geometric_raw_moment_arbitrary_order(m::Sym{ModelingToolkit.Parameter{Real}}, n::Int)
function geometric_raw_moment_arbitrary_order(m::Symbolic, n::Int)
    # computes the symbolic expression of ⟨Xⁿ⟩ where X is a stochastic variable
    # following a geometric distribution with mean burst size m
    # the distribution has the form P(x) = (1-p)ˣp, and m is given by m = (1-p)/p
    #@syms p
    suma = 0.0
    for i in 0:n
        #suma += eulerian_number(n, i) * (1-p)^(n-i)
        suma += eulerian_number(n, i) * m^(n-i) * (1+m)^(i-n)
        suma = simplify(suma)
    end
    #return p^(-n) * suma
    simplify((1+m)^n * suma)
end

#function geometric_raw_moment(m::Sym{ModelingToolkit.Parameter{Real}}, n::Int)
function geometric_raw_moment(m::Symbolic, n::Int)
    # provide the symbolic expression of ⟨Xⁿ⟩ where X is a stochastic variable
    # following a geometric distribution with mean burst size m
    # the distribution has the form P(x) = (1-p)ˣp, and m is given by m = (1-p)/p
    # Although geometric_raw_moment_arbitrary_order() can obtain the moment up to
    # arbitrary order, it is unable to format it in a nice and readable way
    # (not straightforward to use SymbolicUtils.jl to perform a binomial expansion
    # and hence tidy up the expression). For this reason, we provide the clean
    # symbolic expressions up to order 5 just to make the moment equations easier to check
    if n == 0
        1
    elseif n==1
        m
    elseif n==2
        2*m^2+m
    elseif n==3
        m*(6*m^2 + 6*m + 1)
    elseif n==4
        m*(2*m + 1)*(12*m^2+12*m+1)
    elseif n==5
        m*(120*m^4 + 240*m^3 + 150*m^2 + 30*m + 1)
    elseif n>5
        geometric_raw_moment_arbitrary_order(m, n)
    else
        error("negative argument values")
    end
end


# Richardson function
function numdiff(f,x::Number,h::Float64)
    D0 = (f(x+h)-f(x-h))/(2*h);
    D1 = ( f(x+(h/2)) - f(x-(h/2)) )/(h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson function
function numdiff(f,x::Number,h::Int64)
    D0 = (f(x+h)-f(x-h))/(2*h);
    D1 = ( f(x+(h/2)) - f(x-(h/2)) )/(h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function numdiff(x::Array{Float64,1}, y::Array{Float64,1}, val) #We get a x array, a y array and a val in x for which we want to get its derivative
    h = abs(x[2] - x[1]);
    ind = findall(p->p==val, x)[1]
    D0 = (y[ind+2]-y[ind-2])/(4*h);
    D1 = (y[ind+1]-y[ind-1])/(2*h);
    TV = D1 + (D1-D0)/3
    return TV
end

# Richardson getting the whole bunch i.e. CDD of O(h^4) residues

#=
function numdiff(x::Array{Float64,1}, y::Array{Float64,1}) #We get a x array, a y array and a val in x for which we want to get its derivative
    h = abs(x[2] - x[1]);

    return TV
end
=#

# Forward func of linear order
function fdiff(f,x,h)
    d = (f(x+h)-f(x))/h;
    return d
end

# Forward discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function fdiff(x::Array{Float64,1}, y::Array{Float64,1},val)
    h = abs( x[2] - x[1] );
    ind = findall(p->p==val, x)[1]
    d = (y[ind+1] - y[ind])/h
    return d
end

# Forward discrete, with the whole bunch (MATLAB-like, with the whole vector)
function fdiff(x::Array{Float64,1}, y::Array{Float64,1})
    h = abs( x[2] - x[1] );
    d = (y[2:end] - y[1:end-1])/h
    return d
end

# Backwards func
function bdiff(f,x,h)
    d = (f(x)-f(x-h))/h;
    return d
end

# Backwards discrete of type 1 (We get an exact value in x vector in which we want to differentiate)
function bdiff(x::Array{Float64,1}, y::Array{Float64,1},val)
    h = abs( x[2] - x[1] );
    ind = findall(p->p==val, x)[1]
    d = (y[ind] - y[ind-1])/h
    return d
end

# Backwards discrete, with the whole bunch (MATLAB-like, with the whole vector)
function bdiff(x::Array{Float64,1}, y::Array{Float64,1})
    h = abs( x[2] - x[1] );
    d = (y[2:end] - y[1:end-1])/h
    return d
end


using LinearAlgebra
using Statistics

x = [ 2, -1, 2];

# norm of x is written in Julia as norm(x)
norm(x)

sqrt(x'*x)

sqrt(sum(x.^2))

x = randn(10); y = randn(10)

lhs = norm(x+y)

rhs = norm(x) + norm(y)

# rms value of a vector is rms(x) = norm(x)/sqrt(length(x))
rms(x) = norm(x) / sqrt(length(x))

t = 0:0.01:1; # list of times

x = cos.(8*t) - 2*sin.(11*t)

rms(x)

using Plots

plot(t,x)

plot!(t, avg(x)*ones(length(x)))

plot!(t, (avg(x)+rms(x))*ones(length(x)), color = :green)

plot!(t, (avg(x)-rms(x))*ones(length(x)), color = :green)

plot!(legend = false)

# Define Chebyshev bound function
# The Chebyshev inequality states that the number of entries of an
# n-vector x that have absolute value at least a is no more than
# ∥x∥2/a2 = nrms(x)2/a2. If this number is, say, 12.15,
# we can conclude that no more that 12 entries have absolute value
# at least a, since the number of entries is an integer.

cheb_bound(x,a) = floor(norm(x)^2/a);
a = 1.5;
cheb_bound(x,a)
# Number of entries of x with |x_i| >= a
sum(abs.(x) .>= a)


# Algorithm 5.3
# Runge Kutta Fehlberg

function rkf(f, a, b, α, max_step, min_step, tolerance)
	flag = 1
	result = Tuple{Float64, Float64, Float64}[]
	t = a
	k = Array{Float64}(undef, 6)
	h = max_step
	ω = α
	push!(result, (t, ω, h))
	while flag == 1
		k[1] = h*f(t, ω)
		k[2] = h*f(t + (1/4)*h, ω + (1/4)*k[1])
		k[3] = h*f(t + (3/8)*h, ω + (3/32)*k[1] + (9/32)*k[2])
		k[4] = h*f(t + (12/13)*h, ω + (1932/2197)*k[1] - (7200/2197)*k[2] + (7296/2197)*k[3])
		k[5] = h*f(t + h, ω + (439/216)*k[1] - 8*k[2] + (3680/513)*k[3] - (845/4104)*k[4])
		k[6] = h*f(t + (1/2)*h, ω - (8/27)*k[1] + 2*k[2] - (3544/2565)*k[3] + (1859/4104)*k[4] - (11/40)*k[5])
		R = (1/h)*abs((1/360)*k[1] - (128/4275)*k[3] - (2197/75240)*k[4] + (1/50)*k[5] + (2/55)k[6])
		if R <= tolerance
			t = t + h
			ω = ω + (25/216)*k[1] + (1408/25655)*k[3] + (21977/4104)*k[4] - (1/5)*k[5]
			push!(result, (t, ω, h))
		end
		δ = 0.84 * (tolerance/R)^(1/4)
		if δ <= 0.1
			h = 0.1*h
		elseif δ >= 4
			h = 4h
		else
			h = δ*h
		end
		if h > max_step
			h = max_step
		end
		if t >= b
			flag = 0
		elseif (t + h) > b
			h = b - t
		elseif h < min_step
			flag = 0
		end
	end
	return result
end

f(t, y) = y - t^2 + 1
println(rkf(f, 0, 2, 0.5, 0.25, 0, 10^(-5)))


module NewtonMethod2

using LinearAlgebra, Statistics, ForwardDiff

function newtonroot(f, f_prime; x0, tol=1E-7, maxiter=1000)
    # setup the algorithm
    x_old = x0
    normdiff = Inf
    iter = 0
    while normdiff > tol && iter <= maxiter
        x_new = x_old-f(x_old)/f_prime(x_old) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if iter <= maxiter
        return (x_old, normdiff, iter)
    else
        return (nothing, normdiff, iter) # hope it is correct!
    end
end

export newtonroot

#########

function newtonroot2(f; x0, tol=1E-7, maxiter=1000)
    D(f) = x -> ForwardDiff.derivative(f, x)
    f_prime = D(f)
    # setup the algorithm
    x_old = x0
    normdiff = Inf
    iter = 0
    while normdiff > tol && iter <= maxiter
        x_new = x_old-f(x_old)/f_prime(x_old) # use the passed in map
        normdiff = norm(x_new - x_old)
        x_old = x_new
        iter = iter + 1
    end
    if iter <= maxiter
        return (x_old, normdiff, iter)
    else
        return (nothing, normdiff, iter)
    end
end

export(newtonroot2)

end


### A Pluto.jl notebook ###
# v0.17.4

using Markdown
using InteractiveUtils

# ╔═╡ d16725a0-6cd3-11ec-0998-fd4af26f10dd
begin
	using Pkg; Pkg.activate(@__DIR__); Pkg.instantiate()
	Pkg.precompile()

	using PlutoUI
	using Plots
	using Statistics
	using StatsBase
	using PyCall
	using DataFrames
	using GLM
	using Tables
	using XLSX
	using DataStructures
	using MLBase
	using RDatasets
	using LsqFit
end;

# ╔═╡ 91471950-703f-4927-8bce-f4478f4a53ba
PlutoUI.TableOfContents(aside=true, indent=true, depth=3)

# ╔═╡ a53d44c5-75c4-4508-aa19-33e838c9e0bb
md"""
# 📈 Regression

Regression Analysis is a set of statistical processes for estimating the relationships between a _dependent variable_ (often called the _outcome_ or _response_ variable) and one or more _independent variables_ (often called _predictors_, _covariates_, _explanatory variables_ or _features_). The most common form of regression analysis is linear regression, in which one finds the line (or a more complex linear combination) that most closely fits the data according to a specific mathematical criterion.

[YouTube Link](https://www.youtube.com/watch?v=5TCbIK_cpZE)
"""

# ╔═╡ e9458f99-4ec0-4167-b771-3090dfec6320
md"""
## Simple linear regression
"""

# ╔═╡ 784c5b0e-59dc-4b2d-bf40-112261ac8e85
md"""
### Building from scratch

Firstly, let's generate some random data and then plot it...
"""

# ╔═╡ 65e22edc-ef5c-412f-82f4-1c84da209d6d
begin
	# x values
	xvals = repeat(1:0.5:10, inner=2)
	# y values (with some noise)
	yvals = 3 .+ xvals .+ 2 .* rand(length(xvals)) .-1

	# plotting
	scatter(xvals, yvals, color=:black, leg=false,
			xlabel="X values", ylabel="Y values")
end

# ╔═╡ db2c3459-2127-4b46-8e73-546850767dc0
md"""
Assuming an ordinary least squares regression, the slope `a` of the fitted line is equal to the correlation between `y` and `x` corrected by the ratio of standard deviations of these variables. The intercept `b` of the fitted line is such that the line passes through the center of mass (`x`, `y`) of the data points.

Thus, we can easily build the `find_best_fit` function, which returns `a` and `b` coefficients.
"""

# ╔═╡ 3bb8c382-f97d-48dc-8cbe-99bf77de0784
function find_best_fit(x, y)
	X̅, Y̅ = mean(x), mean(y)
	Sˣ, Sʸ = std(x), std(y)

	a = cor(x,y) * (Sˣ/Sʸ)
	b = Y̅ - a*X̅

	return a, b
end

# ╔═╡ 75059ff4-6b9d-4e01-9aed-33ea8b0e9a71
md"""
We can now use the function above to calculate `a` and `b` and then compute the estimated values `ŷvals`...
"""

# ╔═╡ e83331a1-8655-4de0-a36f-b74b5e43674b
begin
	a, b = find_best_fit(xvals, yvals)
	ŷvals₁ = a .* xvals .+ b
end

# ╔═╡ d81e448b-b4e4-4d6f-8ca7-42c41569977d
md"""
### numpy

We can also import the `numpy` python package to perform a simple linear regression. We are able to use any `numpy` function by adding the prefix `np`, just like in Python.
"""

# ╔═╡ a047abfb-178c-40ac-8f8c-c3a8a73c1849
# importing numpy
np = pyimport("numpy");

# ╔═╡ 208af138-5131-4034-885b-c41f6ccf36f5
# numpy model
np_model = np.polyfit(xvals, yvals, 1)

# ╔═╡ 39884fc4-c41c-46e5-a92a-44d766243715
# estimated ŷ values
ŷvals₂ = np_model[1] .* collect(xvals) .+ np_model[2]

# ╔═╡ 13719feb-60ae-433b-bf1e-df6ac045215d
md"""
### GLM

Finally, we can use the [GLM.jl](https://github.com/JuliaStats/GLM.jl) package to perform simple linear regression using the `lm` function.
"""

# ╔═╡ 10c072d4-b793-458c-b947-d5dc3aae5e25
# converting data into a DataFrame
data = DataFrame(X=xvals, Y=yvals);

# ╔═╡ 68a0574f-5b17-48ae-b8c4-a03c7e385561
# GLM model
glm_model = lm(@formula(Y ~ X), data)

# ╔═╡ ecb1cb29-2b57-4197-9327-e762fc876281
# estimated ŷ values
ŷvals₃ = predict(glm_model)

# ╔═╡ 056b28f3-8da4-4a94-a33c-72b30c8aa0aa
md"""
### Comparing methods

Now we can plot all the three models. We can see below that both `numpy` and `GLM` models are exactly the same. The model that we build from scratch is slightly different.
"""

# ╔═╡ 9a5dcad1-9365-423c-b27a-a6c206850848
begin
	# plotting data
	scatter(xvals, yvals, color=:black, label="Data",
			xlabel="X values", ylabel="Y values", legend=:topleft)

	# plotting "from scratch" fitted line
	plot!(xvals, ŷvals₁, color=:red, label="From Scratch")

	# plotting numpy model
	plot!(xvals, ŷvals₂, color=:green, label="Numpy")

	# plotting GLM model
	plot!(xvals, ŷvals₃, color=:blue, label="GLM")
end

# ╔═╡ a89ec73a-5e35-49ad-8829-dc2ecc7e8aac
md"""
## Load real data

Now let's get some real data. We will use housing information from zillow, check out the file `zillow_data_download_april2020.xlsx` for a quick look of what the data looks like. Our goal will be to build a linear regression model between the number of houses listed vs the number of houses sold in a few states. Fitting these models can serve as a key real estate indicator.
"""

# ╔═╡ aa78c706-8860-46f1-9982-ea10bdc4fccb
# loading data
S = XLSX.readxlsx("data/zillow_data_download_april2020.xlsx")

# ╔═╡ 618ab9ee-ee9c-4dbb-b48f-ebeeb15549c6
md"""
**Note:** there are three different sheets in our data.

## Data wrangling

Let's create a `DataFrame` from the `Sale_counts_city` sheet...
"""

# ╔═╡ b530e842-bfa4-4c41-9cc9-4aa629e12f99
# selecting sheet
sale_counts = S["Sale_counts_city"][:]

# ╔═╡ 041e87c7-deee-46c1-85be-ea47df16217c
# converting Matrix into DataFrame
df_sale_counts = DataFrame(sale_counts[2:end,:], Symbol.(sale_counts[1,:]))

# ╔═╡ 19a44596-2367-4964-b21e-7d84ba147915
md"""
Now, we can do the same with the `MonthlyListings_City` sheet...
"""

# ╔═╡ ab8cf8d2-9ac6-4361-b168-243f781e7dfa
begin
	# selecting sheet
	month_list = S["MonthlyListings_City"][:]
	# converting Matrix into DataFrame
	df_month_list = DataFrame(month_list[2:end,:], Symbol.(month_list[1,:]))
end

# ╔═╡ 215483a1-6059-437d-a4b6-ca3cd897f13b
md"""
We may filter just the data from February 2020...
"""

# ╔═╡ b075d090-91aa-4d1f-bb9f-a93dd65413e7
begin
	# monthly listing data (Feb 2020)
	monthlist_2020_2 = df_month_list[!,[1,2,3,4,5,end]]
	rename!(monthlist_2020_2, Symbol("2020-02") => :listing)
end

# ╔═╡ 1b8f7b0d-a843-4af7-90b7-4a64d74edab0
begin
	# sale counts data (Feb 2020)
	salecounts_2020_2 = df_sale_counts[!,[1,end]]
	rename!(salecounts_2020_2, Symbol("2020-02") => :sales)
end

# ╔═╡ 39456bea-24dc-482b-9d8b-33a9f2ed715a
md"""
Next, we can (outer) join our two DataFrames and, then, remove missing values...
"""

# ╔═╡ f195db71-f2a4-4f5c-bfa4-2d9ec1f23a32
begin
	feb2020data = outerjoin(monthlist_2020_2, salecounts_2020_2, on=:RegionID)
	dropmissing!(feb2020data)
end

# ╔═╡ 2da58c4e-8db4-4d80-aa29-f3ec10952f73
md"""
Let's define variables for all the three columns we are interested in...
"""

# ╔═╡ 93179d9a-5bf4-488b-b710-28a5f56b2e26
begin
	sales = feb2020data[!,:sales]
	counts = feb2020data[!,:listing]
	states = feb2020data[!,:StateName]
end;

# ╔═╡ 560a134f-1175-4d56-83af-1ad6bbe1a99f
md"""
Now, we can select just the top 10 states (most cited states). We will use the `counter` function from the [DataStructures.jl](https://github.com/JuliaCollections/DataStructures.jl) package.
"""

# ╔═╡ 0c4a00f6-69f2-4fc3-a7f8-ca6e38404ac9
# counter object (Accumulator)
C = counter(states)

# ╔═╡ a02a9fdf-46bc-4ea1-bb29-b1731a66c057
# converting into a Dictionary
Cdict = C.map

# ╔═╡ 63b596be-9392-4efb-8d31-7f1d6e9c7254
begin
	# getting counts as an Array
	countvals = values(Cdict) |> collect

	# getting state names as an Array
	state_labels = keys(Cdict) |> collect
end;

# ╔═╡ feced955-2e69-44ae-bc2a-82d4f2136b38
# getting indexes from 10 top states
topstateindexes = sortperm(countvals, rev=true)[1:10]

# ╔═╡ 831484f5-6cdc-4641-8adf-117a33c518c2
md"""
**Note:** the built-in `sortperm` function returns a permutation vector `I` that puts `v[I]` in sorted order. The order is specified using the same keywords as `sort!`.
"""

# ╔═╡ b94019d8-a7d8-4781-819b-28eeac823812
# getting top 10 states names
topstates = state_labels[topstateindexes]

# ╔═╡ 4735fc87-68cb-4617-aa7c-6139ba7b7c0a
md"""
## Linear regression models

Firstly, we will plot the model for each state using the formula `Y ~ X`.
"""

# ╔═╡ ec7ae9e6-b191-4578-8a47-9d2245280316
begin
	# creating a blank Array of Plot
	all_plots₁ = Array{Plots.Plot,1}(undef,10)

	for (i,s) in enumerate(topstates)
		# finding indexes of all occurences of a given state
		ids = findall(feb2020data[!,:StateName] .== s)
		# getting data for a given state
		data = DataFrame(X=Float64.(counts[ids]), Y=Float64.(sales[ids]))
		# training the model
		ols = GLM.lm(@formula(Y ~ X), data)

		# plotting data by each state
		all_plots₁[i] = scatter(data.X, data.Y, markersize=2,
								xlims=(0,500), ylims=(0,500),
								color=i, aspect_ratio=:equal,
								legend=false, title=s)

		# plotting model by each state
		plot!(data.X, predict(ols), color=:black)
	end

	# display plots
	plot(all_plots₁..., layout=(2,5), size=(850,300))
end

# ╔═╡ 216dc17e-54c1-4520-890d-8663165b10dd
md"""
We can do the same thing, but now we will plot all the elements within the same figure.

**Note:** when we use the `Y ~ X + 0` formula, it forces the intercept to be zero.
"""

# ╔═╡ dddfb8fb-7047-4417-8dd2-569a93277f0c
begin
	plot()

	for (i,s) in enumerate(topstates)
	    ids = findall(feb2020data[!,:StateName] .== s)
	    data = DataFrame(X=Float64.(counts[ids]), Y=Float64.(sales[ids]))
	    ols = GLM.lm(@formula(Y ~ 0 + X), data)

	    scatter!(counts[ids], sales[ids], markersize=2, xlim=(0,500),
				 ylim=(0,500), color=i, aspect_ratio=:equal,
	        	 legend=false, marker=(3,3,stroke(0)), alpha=0.2)

	    if s ∈ ["NC","CA","FL"]
	        annotate!([(500-20,10+coef(ols)[1]*500, text(s,10))])
	    end

	    plot!(counts[ids], predict(ols), color=i, linewidth=2)
	end

	xlabel!("Listings")
	ylabel!("Sales")
end

# ╔═╡ 5116c117-75e4-4989-b06f-2883bba48330
md"""
## Logistic regression

So far, we have shown several ways to solve the linear regression problem in Julia. Here, we will first start with a motivating example of when you would want to use logistic regression. Let's assume that our predictor vector is binary (`0` or `1`), let's fit a linear regression model.
"""

# ╔═╡ 3a86069b-3f24-4eec-a7ae-f77189a4fd77
# generating random data
𝒮 = DataFrame(X=collect(1:7), Y=[1,0,1,1,1,1,1])

# ╔═╡ 453385dd-8bda-42b8-9b4a-6a4963b21346
# linear regression model
lin_reg = lm(@formula(Y ~ X), 𝒮)

# ╔═╡ bd31693d-dee9-4534-b4d4-3904997e8020
begin
	# plotting data
	scatter(𝒮.X, 𝒮.Y, legend=false, xlabel="X", ylabel="Y")
	#plotting model
	plot!(1:7, predict(lin_reg))
end

# ╔═╡ 0fb142cd-ac45-42e8-b5d1-24c6142392bb
md"""
What this plot quickly shows is that linear regression may end up predicting values outside the `[0,1]` interval. For an example like this, we will use logistic regression. Interestingly, a generalized linear model (https://en.wikipedia.org/wiki/Generalized_linear_model) unifies concepts like linear regression and logistic regression, and the [GLM.jl](https://github.com/JuliaStats/GLM.jl) package allows you to apply either of these regressions easily by specifying the `distribution family` and the `link` function.

To apply logistic regression via the `GLM.jl` package, you can readily use the `Binomial` family and the `LogitLink` link function.

### Load data

Firstly, let's load some data from the [RDatasets.jl](https://github.com/JuliaStats/RDatasets.jl) package...
"""

# ╔═╡ 3caa5331-2507-4299-b2e4-50e8a5cbbe46
cats = dataset("MASS", "cats")

# ╔═╡ 22c201af-9437-4dde-8653-508ebf514008
md"""
### Data wrangling

We will map the sex of each cat to a binary `0/1` value.
"""

# ╔═╡ 16367b7a-65ea-4fa0-abad-a1cc25c60ee6
# getting the map
sex_map = labelmap(cats.Sex)

# ╔═╡ 1d7ef9db-dd15-4ed4-841a-b4baf02302a1
# encoding Sex column
sex_enconde = labelencode(sex_map, cats.Sex)

# ╔═╡ f42bdc3a-943b-4157-853d-860af52e059c
# plotting data by Sex
scatter(cats.BWt, cats.HWt, color=sex_enconde, legend=false)

# ╔═╡ 310f2282-d6dd-405d-8399-e75fdc75c27c
md"""
Females (blue) seem to be more present in the lower left corner and Males (orange) seem to be present in the top right corner.

We will use the `HWt` variable as the only feature and the `encoded_sex` as the target. Let's convert labels from `1/2` into `0/1`...
"""

# ╔═╡ cc911a53-9139-4c90-b02b-15f30e6772f0
cats_df = DataFrame(X=cats[!,:HWt], Y=sex_enconde .- 1)

# ╔═╡ 79bbe213-c15a-4ecb-aef7-107846234f1d
md"""
### Modeling data

Let's run a logistic regression model on this data.
"""

# ╔═╡ 11b1f5d1-7ae2-43dd-bd35-5b0ba6fde6f4
# logistic model
probit = glm(@formula(Y ~ X), cats_df, Binomial(), LogitLink())

# ╔═╡ e3e9d570-af57-4065-84ec-b61400fa449d
begin
	# plotting ground truth data
	scatter(cats_df.X, cats_df.Y, label="Ground Truth", color=6)

	# plotting predictions
	scatter!(cats_df.X, predict(probit), label="Predictions", color=7,
			 xlabel="Heart Rate", ylabel="Pr(Sex=M)")
end

# ╔═╡ dfc76a4c-4e2b-4588-92e1-2209507fd670
md"""
**Note:** as you can see, contrary to the linear regression case, the predicted values do not go beyond `1`.
"""

# ╔═╡ 31308555-dfc9-4e28-9108-8a83d65b7770
md"""
## Non-linear regression

Finally, sometimes you may have a set of points and the goal is to fit a non-linear function (maybe a quadratic function, a cubic function, an exponential function...). The way we would solve such a problem is by minimizing the least square error between the fitted function and the observations we have. We will use the package [LsqFit.jl](https://github.com/JuliaNLSolvers/LsqFit.jl) for this task.

**Note:** this problem is usually modeled as a numerical optimizaiton problem.
"""

# ╔═╡ 4a7d2c25-04b1-4ccb-85cd-7276902a9679
md"""
### Generate data

We will first set up our data and plot it...
"""

# ╔═╡ 7203ba1b-ce21-4c53-9dce-358df13df21e
begin
	x = 0:0.05:10
	y = 1 * exp.(-x * 2) + 2 * sin.(0.8 * π * x) + 0.15 * randn(length(x));
	scatter(x, y, legend=false)
end

# ╔═╡ 775e6fbe-8c10-4f94-b53e-618e4dd551fc
md"""
### Modeling data

Then, we set up the model with `model(x,p)`. The vector `p` is what to be estimated given a set of values `x`.
"""

# ╔═╡ 5d75ebce-1181-4490-ae10-8bcc08456fdf
begin
	@. model(v, p) = p[1] * exp(-v * p[2]) + p[3] * sin(0.8 * π *v)
	p0 = [0.5, 0.5, 0.5]
	myfit = curve_fit(model, x, y, p0)
end

# ╔═╡ a94163fe-e1ac-483a-bc2d-5d7795cfe188
md"""
⚠ A note about `curve_fit`: this function can take multiple other inputs, for instance the Jacobian of what you are trying to fit. We don't dive into these details here, but be sure to check out the LsqFit.jl package to see what other things you can can pass to create a better fit.

Also note that julia has multiple packages that allow you to create Jacobians so you don't have to write them yourself. Two such packages are [FiniteDifferences.jl](https://github.com/JuliaDiff/FiniteDifferences.jl) or [ForwardDiff.jl](https://github.com/JuliaDiff/ForwardDiff.jl).

↪ Back to our example. Let's define the paramenter variable and then calculate the predictions `ŷ`...
"""

# ╔═╡ 6a965e6d-4724-47c8-a3bb-bfec7d6706c9
begin
	p = myfit.param
	ŷ = p[1] * exp.(-x * p[2]) + p[3] * sin.(0.8 * π * x)
end

# ╔═╡ c7d07a47-3f3f-482e-9c06-78877b91e380
md"""
We are ready now to plot the curve we have generated...
"""

# ╔═╡ 6ca3290d-5980-4749-b309-fd84d73b7bb5
begin
	scatter(x, y, legend=false, xlabel="X", ylabel="Y")
	plot!(x, ŷ)
end

# ╔═╡ Cell order:
# ╟─d16725a0-6cd3-11ec-0998-fd4af26f10dd
# ╟─91471950-703f-4927-8bce-f4478f4a53ba
# ╟─a53d44c5-75c4-4508-aa19-33e838c9e0bb
# ╟─e9458f99-4ec0-4167-b771-3090dfec6320
# ╟─784c5b0e-59dc-4b2d-bf40-112261ac8e85
# ╠═65e22edc-ef5c-412f-82f4-1c84da209d6d
# ╟─db2c3459-2127-4b46-8e73-546850767dc0
# ╠═3bb8c382-f97d-48dc-8cbe-99bf77de0784
# ╟─75059ff4-6b9d-4e01-9aed-33ea8b0e9a71
# ╠═e83331a1-8655-4de0-a36f-b74b5e43674b
# ╟─d81e448b-b4e4-4d6f-8ca7-42c41569977d
# ╠═a047abfb-178c-40ac-8f8c-c3a8a73c1849
# ╠═208af138-5131-4034-885b-c41f6ccf36f5
# ╠═39884fc4-c41c-46e5-a92a-44d766243715
# ╟─13719feb-60ae-433b-bf1e-df6ac045215d
# ╠═10c072d4-b793-458c-b947-d5dc3aae5e25
# ╠═68a0574f-5b17-48ae-b8c4-a03c7e385561
# ╠═ecb1cb29-2b57-4197-9327-e762fc876281
# ╟─056b28f3-8da4-4a94-a33c-72b30c8aa0aa
# ╟─9a5dcad1-9365-423c-b27a-a6c206850848
# ╟─a89ec73a-5e35-49ad-8829-dc2ecc7e8aac
# ╠═aa78c706-8860-46f1-9982-ea10bdc4fccb
# ╟─618ab9ee-ee9c-4dbb-b48f-ebeeb15549c6
# ╠═b530e842-bfa4-4c41-9cc9-4aa629e12f99
# ╠═041e87c7-deee-46c1-85be-ea47df16217c
# ╟─19a44596-2367-4964-b21e-7d84ba147915
# ╠═ab8cf8d2-9ac6-4361-b168-243f781e7dfa
# ╟─215483a1-6059-437d-a4b6-ca3cd897f13b
# ╠═b075d090-91aa-4d1f-bb9f-a93dd65413e7
# ╠═1b8f7b0d-a843-4af7-90b7-4a64d74edab0
# ╟─39456bea-24dc-482b-9d8b-33a9f2ed715a
# ╠═f195db71-f2a4-4f5c-bfa4-2d9ec1f23a32
# ╟─2da58c4e-8db4-4d80-aa29-f3ec10952f73
# ╠═93179d9a-5bf4-488b-b710-28a5f56b2e26
# ╟─560a134f-1175-4d56-83af-1ad6bbe1a99f
# ╠═0c4a00f6-69f2-4fc3-a7f8-ca6e38404ac9
# ╠═a02a9fdf-46bc-4ea1-bb29-b1731a66c057
# ╠═63b596be-9392-4efb-8d31-7f1d6e9c7254
# ╠═feced955-2e69-44ae-bc2a-82d4f2136b38
# ╟─831484f5-6cdc-4641-8adf-117a33c518c2
# ╠═b94019d8-a7d8-4781-819b-28eeac823812
# ╟─4735fc87-68cb-4617-aa7c-6139ba7b7c0a
# ╠═ec7ae9e6-b191-4578-8a47-9d2245280316
# ╟─216dc17e-54c1-4520-890d-8663165b10dd
# ╠═dddfb8fb-7047-4417-8dd2-569a93277f0c
# ╟─5116c117-75e4-4989-b06f-2883bba48330
# ╠═3a86069b-3f24-4eec-a7ae-f77189a4fd77
# ╠═453385dd-8bda-42b8-9b4a-6a4963b21346
# ╠═bd31693d-dee9-4534-b4d4-3904997e8020
# ╟─0fb142cd-ac45-42e8-b5d1-24c6142392bb
# ╠═3caa5331-2507-4299-b2e4-50e8a5cbbe46
# ╟─22c201af-9437-4dde-8653-508ebf514008
# ╠═16367b7a-65ea-4fa0-abad-a1cc25c60ee6
# ╠═1d7ef9db-dd15-4ed4-841a-b4baf02302a1
# ╠═f42bdc3a-943b-4157-853d-860af52e059c
# ╟─310f2282-d6dd-405d-8399-e75fdc75c27c
# ╠═cc911a53-9139-4c90-b02b-15f30e6772f0
# ╟─79bbe213-c15a-4ecb-aef7-107846234f1d
# ╠═11b1f5d1-7ae2-43dd-bd35-5b0ba6fde6f4
# ╠═e3e9d570-af57-4065-84ec-b61400fa449d
# ╟─dfc76a4c-4e2b-4588-92e1-2209507fd670
# ╟─31308555-dfc9-4e28-9108-8a83d65b7770
# ╟─4a7d2c25-04b1-4ccb-85cd-7276902a9679
# ╠═7203ba1b-ce21-4c53-9dce-358df13df21e
# ╟─775e6fbe-8c10-4f94-b53e-618e4dd551fc
# ╠═5d75ebce-1181-4490-ae10-8bcc08456fdf
# ╟─a94163fe-e1ac-483a-bc2d-5d7795cfe188
# ╠═6a965e6d-4724-47c8-a3bb-bfec7d6706c9
# ╟─c7d07a47-3f3f-482e-9c06-78877b91e380
# ╠═6ca3290d-5980-4749-b309-fd84d73b7bb5


# Classical Multidimensional Scaling

## convert Gram matrix to Distance matrix

"""
    gram2dmat!(D, G)

Convert a Gram matrix `G` to a distance matrix, and write the results to `D`.
"""
function gram2dmat!(D::AbstractMatrix{DT}, G::AbstractMatrix) where {DT<:Real}
    # argument checking
    m = size(G, 1)
    n = size(G, 2)
    m == n || error("D should be a square matrix.")
    size(D) == (m, n) ||
        throw(DimensionMismatch("Sizes of D and G do not match."))

    # implementation
    for j = 1:n
        for i = 1:j-1
            @inbounds D[i,j] = D[j,i]
        end
        D[j,j] = zero(DT)
        for i = j+1:n
            @inbounds D[i,j] = sqrt(G[i,i] + G[j,j] - 2 * G[i,j])
        end
    end
    return D
end

"""
    gram2dmat(G)

Convert a Gram matrix `G` to a distance matrix.
"""
gram2dmat(G::AbstractMatrix{T}) where {T<:Real} = gram2dmat!(similar(G, T), G)

## convert Distance matrix to Gram matrix

"""
    dmat2gram!(G, D)

Convert a distance matrix `D` to a Gram matrix, and write the results to `G`.
"""
function dmat2gram!(G::AbstractMatrix{GT}, D::AbstractMatrix) where GT
    # argument checking
    n = LinearAlgebra.checksquare(D)
    size(G) == (n, n) ||
        throw(DimensionMismatch("Sizes of G and D do not match."))

    # implementation
    u = zeros(GT, n)
    s = 0.0
    for j = 1:n
        s += (u[j] = sum(abs2, view(D,:,j)) / n)
    end
    s /= n

    for j = 1:n
        for i = 1:j-1
            @inbounds G[i,j] = G[j,i]
        end
        for i = j:n
            @inbounds G[i,j] = (u[i] + u[j] - abs2(D[i,j]) - s) / 2
        end
    end
    return G
end

momenttype(T) = typeof((zero(T) * zero(T) + zero(T) * zero(T))/ 2)

"""
    dmat2gram(D)

Convert a distance matrix `D` to a Gram matrix.
"""
dmat2gram(D::AbstractMatrix{T}) where {T<:Real} = dmat2gram!(similar(D, momenttype(T)), D)

## Classical MDS

"""
*Classical Multidimensional Scaling* (MDS), also known as Principal Coordinates Analysis (PCoA),
is a specific technique in this family that accomplishes the embedding in two steps:

1. Convert the distance matrix to a Gram matrix. This conversion is based on
the following relations between a distance matrix ``D`` and a Gram matrix ``G``:

```math
\\mathrm{sqr}(\\mathbf{D}) = \\mathbf{g} \\mathbf{1}^T + \\mathbf{1} \\mathbf{g}^T - 2 \\mathbf{G}
```

Here, ``\\mathrm{sqr}(\\mathbf{D})`` indicates the element-wise square of ``\\mathbf{D}``,
and ``\\mathbf{g}`` is the diagonal elements of ``\\mathbf{G}``. This relation is
itself based on the following decomposition of squared Euclidean distance:

```math
\\| \\mathbf{x} - \\mathbf{y} \\|^2 = \\| \\mathbf{x} \\|^2 + \\| \\mathbf{y} \\|^2 - 2 \\mathbf{x}^T \\mathbf{y}
```

2. Perform eigenvalue decomposition of the Gram matrix to derive the coordinates.

*Note:*  The Gramian derived from ``D`` may have non-positive or degenerate
eigenvalues.  The subspace of non-positive eigenvalues is projected out
of the MDS solution so that the strain function is minimized in a
least-squares sense.  If the smallest remaining eigenvalue that is used
for the MDS is degenerate, then the solution is not unique, as any
linear combination of degenerate eigenvectors will also yield a MDS
solution with the same strain value.
"""
struct MDS{T<:Real} <: NonlinearDimensionalityReduction
    d::Real                  # original dimension
    X::AbstractMatrix{T}     # fitted data, X (d x n)
    λ::AbstractVector{T}     # eigenvalues in feature space, (k x 1)
    U::AbstractMatrix{T}     # eigenvectors in feature space, U (n x k)
end

## properties
"""
    size(M)

Returns tuple where the first value is the MDS model `M` input dimension,
*i.e* the dimension of the observation space, and the second value is the output
dimension, *i.e* the dimension of the embedding.
"""
size(M::MDS) = (M.d, size(M.U,2))

"""
    projection(M)

Get the MDS model `M` eigenvectors matrix (of size ``(n, p)``) of the embedding space.
The eigenvectors are arranged in descending order of the corresponding eigenvalues.
"""
projection(M::MDS) = M.U

"""
    eigvecs(M::MDS)

Get the MDS model `M` eigenvectors matrix.
"""
eigvecs(M::MDS) = projection(M)

"""
    eigvals(M::MDS)

Get the eigenvalues of the MDS model `M`.
"""
eigvals(M::MDS) = M.λ

"""
    loadings(M::MDS)

Get the loading of the MDS model `M`.
"""
loadings(M::MDS) = sqrt.(M.λ)' .* M.U

## use

"""
    predict(M, x::AbstractVector)

Calculate the out-of-sample transformation of the observation `x` for the MDS model `M`.
Here, `x` is a vector of length `d`.
"""
function predict(M::MDS, x::AbstractVector{T}; distances=false) where {T<:Real}
    d = if isnan(M.d) # model has only distance matrix
        @assert distances "Cannot transform points if model was fitted with a distance matrix. Use point distances."
        size(x, 1) != size(M.X, 1) && throw(
            DimensionMismatch("Point distances should be calculated to all original points"))
        x
    else
        if distances
            size(x, 1) != size(M.X, 2) && throw(
                DimensionMismatch("Point distances should be calculated to all original points."))
            x
        else
            size(x, 1) != size(M.X, 1) && throw(
                DimensionMismatch("Points and original data must have same dimensionality."))
            pairwise((x,y)->norm(x-y), M.X, x)
        end
    end

    # get distance matrix
    D = isnan(M.d) ? M.X : pairwise((x,y)->norm(x-y), eachcol(M.X), symmetric=true)
    d = d.^2

    # b = 0.5*(ones(n,n)*d./n - d + D*ones(n,1)./n - ones(n,n)*D*ones(n,1)./n^2)
    mD = mean(D.^2, dims=2)
    b = (d  .- mean(d, dims=1) .- mD .+ mean(mD)) ./ -2

    # sqrt(λ)⁻¹U'b
    λ = vcat(M.λ, zeros(T, size(M)[2] - length(M.λ)))
    return M.U' * b ./ sqrt.(λ)
end

"""
    predict(M)

Returns a coordinate matrix of size ``(p, n)`` for the MDS model `M`, where each column
is the coordinates for an observation in the embedding space.
"""
function predict(M::MDS{T}) where {T<:Real}
    d, p = size(M)
    # if there are non-positive missing eigval then pad with zeros
    λ = vcat(M.λ, zeros(T, p - length(M.λ)))
    return diagm(0=>sqrt.(λ)) * M.U'
end

## show

function show(io::IO, M::MDS)
    d, p = size(M)
    print(io, "Classical MDS(indim = $d, outdim = $p)")
end

## interface functions

"""
    fit(MDS, X; kwargs...)

Compute an embedding of `X` points by classical multidimensional scaling (MDS).
There are two calling options, specified via the required keyword argument `distances`:

    mds = fit(MDS, X; distances=false, maxoutdim=size(X,1)-1)

where `X` is the data matrix. Distances between pairs of columns of `X` are computed using the Euclidean norm.
This is equivalent to performing PCA on `X`.

    mds = fit(MDS, D; distances=true, maxoutdim=size(D,1)-1)

where `D` is a symmetric matrix `D` of distances between points.
"""
function fit(::Type{MDS}, X::AbstractMatrix{T};
             maxoutdim::Int = size(X,1)-1,
             distances::Bool) where T<:Real

    # get distance matrix and space dimension
    D, d = if !distances
        pairwise((x,y)->norm(x-y), eachcol(X), symmetric=true), size(X,1)
    else
        X, NaN
    end
    G = dmat2gram(D)

    n = size(D, 1)
    m = min(maxoutdim, n) #Actual number of eigenpairs wanted

    E = eigen!(Hermitian(G))

    #Sometimes dmat2gram produces a negative definite matrix, and the sign just
    #needs to be flipped. The heuristic to check for this robustly is to check
    #if there is a negative eigenvalue of magnitude larger than the largest
    #positive eigenvalue, and flip the sign of eigenvalues if necessary.
    mineig, maxeig = extrema(E.values)
    if mineig < 0 && abs(mineig) > abs(maxeig)
        #do flip
        ord = sortperm(E.values)
        λ = -E.values[ord[1:m]]
    else
        ord = sortperm(E.values; rev=true)
        λ = E.values[ord[1:m]]
    end

    for i in 1:m
        if λ[i] <= 0
            #Keeping all remaining eigenpairs would not change solution (if 0)
            #or make the answer _worse_ (if <0).
            #The least squares solution would want to throw all these away.
            @warn("Gramian has only $(i-1) positive eigenvalue(s)")
            m = i-1
            ord = ord[1:m]
            λ = λ[1:m]
            break
        end
    end

    #Check if the last considered eigenvalue is degenerate
    if m>0
        nevalsmore = sum(abs.(E.values[ord[m+1:end]] .- λ[m]) .< n*eps())
        nevals = sum(abs.(E.values .- λ[m]) .< n*eps())
        if nevalsmore > 1
            @warn("The last eigenpair is degenerate with $(nevals-1) others; $nevalsmore were ignored. Answer is not unique")
        end
    end
    U = E.vectors[:, ord[1:m]]

    #Add trailing zero coordinates if dimension of embedding space (p) exceeds
    #number of eigenpairs used (m)
    if m < maxoutdim
        U = [U zeros(T, n, maxoutdim-m)]
    end

    return MDS(d, X, λ, U)
end

"""
    stress(M)

Get the stress of the MDS mode `M`.
"""
function stress(M::MDS)
    # calculate distances if original data was stored
    DX = isnan(M.d) ? M.X : pairwise((x,y)->norm(x-y), eachcol(M.X), symmetric=true)
    DY = pairwise((x,y)->norm(x-y), eachcol(predict(M)), symmetric=true)
    n = size(DX,1)
    return sqrt(2*sum((DX - DY).^2)/sum(DX.^2));
end


export minkowski_distance, euclidian_distance, manhattan_distance, chebyshev_distance, braycurtis_distance, canberra_distance, cityblock_distance, mahalanobis_distance,
correlation, cosine_distance, cosine_similarity

using LinearAlgebra
using Statistics: mean

#TODO
# scikitlearn-like kernels ?

# Distance methods

"""
```minkowski_distance(u,v; keywords)```
```minkowski_distance(x::Tuple; keywords)```

Return the Minkowski distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `p::Int = 2` : p value
- `w::Int` : weights that must be of the same length with u,v
- `rooted = false` : denotes whether or not to execute the final rooting operation
"""
function minkowski_distance(u,v; p = 2, w = nothing, rooted = false)
    _validate_distance_input(u, v, w; p = p, p_is_used = true)
    val = abs.(u .- v)
    val = w != nothing ? val .* w : val
    if p == 1
        val = sum(val)
    elseif p == Inf
        val = maximum(val)
    else
        val = sum(val .^ p)
    end
    return rooted ? val .^ (1 / p) : val
end

minkowski_distance(x::Tuple; p = 2, w = nothing) = minkowski_distance(x...; p = p, w = w)

"""
```euclidian_distance(u,v; keywords)```
```euclidian_distance(x::Tuple; keywords)```

Return the Euclidian distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
- `squared = false` : denotes whether or not to execute the final squaring operation
"""
euclidian_distance(u,v; w = nothing, squared = false) = minkowski_distance(u,v; w = w, p = 2, rooted = squared)
euclidian_distance(x::Tuple;w = nothing, squared = false) = euclidian_distance(x...; w = w, rooted = squared)

"""
```manhattan_distance(u,v; keywords)```
```manhattan_distance(x::Tuple; keywords)```

Return the Manhattan distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
"""
manhattan_distance(u,v; w = nothing) = minkowski_distance(u,v; w = w, p = 1)
manhattan_distance(x::Tuple;w = nothing) = manhattan_distance(x...; w = w)

"""
```chebyshev_distance(u,v; keywords)```
```chebyshev_distance(x::Tuple; keywords)```

Return the Chebyshev distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
"""
chebyshev_distance(u,v; w = nothing) = minkowski_distance(u,v; w = w, p = Inf, rooted=false)
chebyshev_distance(x::Tuple;w = nothing) = chebyshev_distance(x...; w = w)

"""
```braycurtis_distance(u,v; keywords)```
```braycurtis_distance(x::Tuple; keywords)```

Return the Braycurtis distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
"""
function braycurtis_distance(u,v; w = nothing)
    _validate_distance_input(u, v, w)
    difference = abs.(u .- v)
    summation = abs.(u .+ v)
    if w != nothing
        difference = w .* difference
        summation = w .* summation
    end
    sum(difference) / sum(summation)
end

braycurtis_distance(x::Tuple; w = nothing) = braycurtis_distance(x...; w = w)

"""
```canberra_distance(u,v; keywords)```
```canberra_distance(x::Tuple; keywords)```

Return the Canberra distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
"""
function canberra_distance(u,v; w = nothing)
    _validate_distance_input(u, v, w)
    difference = abs.(u .- v)
    result = difference / (abs.(u) .+ abs.(v))
    if w != nothing
        result = result .* w
    end
    return  sum(result[result .!== Inf])
end
canberra_distance(x::Tuple; w = nothing) = canberra_distance(x...; w = w)

"""
```cityblock_distance(u,v; keywords)```
```cityblock_distance(x::Tuple; keywords)```

Return the Cityblock distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
"""
function cityblock_distance(u,v; w = nothing)
    _validate_distance_input(u,v,w)
    result = abs.(u .- v)
    if w != nothing
        result = result .* w
    end
    return sum(result)
end
cityblock_distance(x; w = nothing) = cityblock_distance(x...; w = w)

"""
```mahalanobis_distance(u,v; keywords)```
```mahalanobis_distance(x::Tuple; keywords)```

Return the Cityblock distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)
- `Vinv::Matrix` : The inverse of the covariance matrix.

"""
function mahalanobis_distance(u,v, Vinv)
    _validate_distance_input(u,v,nothing)
    delta = u .- v
    return sqrt.(dot(dot(delta, convert_1d(Vinv)), delta))
end

mahalanobis_distance(x::Tuple, Vinv) = mahalanobis_distance(x..., Vinv)

##

"""
```correlation(u,v; keywords)```
```correlation(x::Tuple; keywords)```

Return the Cityblock distance between the given points both of which are given with vector-like structures.

## Arguments
- `u` : first point
- `v` : second point
- `x::Tuple` : tuple as follows: (u,v)

## Keywords
- `w::Int` : weights that must be of the same length with u,v
- `centered=true` : If true, u and v will be centered.
"""
function correlation(u,v; w = nothing, centered = true)
    _validate_distance_input(u,v,w)
    w = w != nothing ? w : ones(length(u))
    _u = convert_1d(u)
    _v = convert_1d(v)
    if centered
        umu = mean(_u .* w)
        vmu = mean(_v .* w)
        _u = _u .- umu
        _v = _v .- vmu
    end
    uv = mean(_u .* _v .* w)
    uu = mean( (_u .^ 2) .* w )
    vv = mean( (_v .^ 2) .* w  )
    return abs(1 - uv / sqrt(uu*vv))
end

cosine_distance(u,v;w=nothing) = correlation(u,v; w = w)
cosine_similarity(u,v; w = nothing) = -cosine_distance(u,v; w = w) + 1


"""
A 2D Rotation matrix is a mtrix that rotates a vector in a 2D real space by an angle theta.
For more info: https://en.wikipedia.org/wiki/Rotation_matrix

This function takes the angle `theta` in radians as input and returns a 2D Matrix which will rotate the the vector by angle `theta`.
"""
function rotation_matrix(θ)
	rot_mat = Matrix{Float64}(undef,2,2)
	rot_mat[1,1]  = cos(θ)
	rot_mat[1,2] = -sin(θ)
	rot_mat[2,1] = sin(θ)
	rot_mat[2,2] = cos(θ)

	return rot_mat
end


#=
Please see
https://computationalmindset.com/en/neural-networks/ordinary-differential-equation-solvers.html#ode2
for details
=#

#=Original problem
x′′ + x′ + 2x = 0
x(0)=1
x′(0)=0
=#

#=Esplicit form of equation
x′′ = -x′ - 2x
=#

#=Analytical solution
x(t) = e^(-t/2) (cos(√7 t / 2) + sin(√7 t / 2) / √7)
=#

using DifferentialEquations
using Plots

function ode_fn(dx,x,p,t)
    -dx -2.0 * x
end

an_sol(t) = exp(-t/2.0) *
            (cos(sqrt(7.0) * t / 2.0) + sin(sqrt(7.0) * t / 2.0)/sqrt(7.0))

t_begin=0.0
t_end=12.0
tspan = (t_begin,t_end)
x_init=1.0
dxdt_init=0.0

prob = SecondOrderODEProblem(ode_fn, dxdt_init, x_init, tspan)
num_sol = solve(prob, Tsit5(), reltol=1e-8, abstol=1e-8)
x_num_sol = [u[2] for u in num_sol.u]

plot(num_sol.t, an_sol.(num_sol.t),
    linewidth=2, ls=:dash,
    title="ODE 2nd order IVP solved by D.E. package",
    xaxis="t", yaxis="x",
    label="analytical",
    legend=true)
plot!(num_sol.t, x_num_sol,
    linewidth=1,
    label="numerical")


## The birthday paradox in theory and in practice.

## Let's simulate the classic birthday paradox.
# In this simulation, we will draw birthdays at random for a group
# of N people, and see if there are any two birthdays that
# are the same.

"""
N is the number of people

This function returns true if two people had the same birthday.
"""
function simple_birthday_simulation(N::Int)
    birthdays = rand(1:365, N)
    # so each birthday is an integer from 1 to 365.
    # we need to check if all of the birthdays are unique
    return !allunique(birthdays) # ! reverses true -> false, false -> true
end

simple_birthday_simulation(20)

## Let's look at what happens as we vary the number of people.
ntrials = 100
prob_dup_birthday = zeros(0) # this is an empty array
for people=5:40
    numdup = 0
    for t=1:ntrials
        numdup += simple_birthday_simulation(people)
    end
    push!(prob_dup_birthday, numdup/ntrials)
end
# make a quick people - by - prob table
display([5:40 prob_dup_birthday])
# but also make a plot
using Plots
plot(5:40, prob_dup_birthday)
xlabel!("People")
ylabel!("Prob. of shared birthday")

## Some ideas to improve
# IDEA -- add confidence intervals
# IDEA -- add "exact" curve based on prob.

## Let's use real data.
using DelimitedFiles
birthdays = readdlm("1-julia-intro/birthday.txt")
plot(birthdays[:,2])

## First step, let's simulate a random birthday from data.
using StatsBase
bday_weights = fweights(Int.(birthdays[:,2]))
sample(bday_weights)

## Let's do the same demo
sample_bdays = [sample(fweights([2,4,3,1])) for i=1:1000]
sum(sample_bdays .== 4)

## Let's implement the sampling ourselves too...
"""
N is the number of people
data is an array of length number of days with the number of people
that were born on that day.

Internally, what this does is convert data to a cum. prob. distribution
and then we use a random [0,1] sample to find where in the CDF we fall.

The idea of how this works is:
give probs [0.2 0.4 0.3 0.1] (i.e. we have 4 birthdays)
then we are going to make an array
[0.2, 0.6, 0.9, 1.0]
and then we are going to guess a random [0,1] variable
and see which bin it falls into.
if the rv. is less than 0.2, we will call it birthday 1
if the rv. is less between 0.2, 0.6, we will call it birthday 2
if the rv. is less between 0.6, 0.9, we will call it birthday 3
if the rv. is less between 0.9, 1, we will call it birthday 4
but this will work for even larger lists too.
"""
function sample_birthdays(N::Int, data)
    probs = data/sum(data)
    cumdist = cumsum(probs)
    bdays = zeros(Int, N)
    for i=1:N
        rv = rand()
        for j=1:length(cumdist)
            if rv <= cumdist[j]
                bdays[i] = j
                break # need to stop when we find the birthday
            end
        end
    end
    return bdays
end
bdays = sample_birthdays(1000, [2,4,3,1])
# Let's check their frequency
sum(bdays .== 3) # count the number of "2" birthdays in our test
## IDEA - use binary search to find the right birthday faster.

## Let's simulate the data-based shared birthday probability
function data_birthday_simulation(N::Int, data)
    birthdays = sample_birthdays(N, data)
    # so each birthday is an integer from 1 to 365.
    # we need to check if all of the birthdays are unique
    return !allunique(birthdays) # ! reverses true -> false, false -> true
end

data_birthday_simulation(20, Int.(birthdays[:,2]))

ntrials = 100
prob_dup_birthday = zeros(0) # this is an empty array
for people=5:40
    numdup = 0
    for t=1:ntrials
        numdup += data_birthday_simulation(people, Int.(birthdays[:,2]))
    end
    push!(prob_dup_birthday, numdup/ntrials)
end
# make a quick people - by - prob table
display([5:40 prob_dup_birthday])
# but also make a plot
using Plots
plot(5:40, prob_dup_birthday)
xlabel!("People")
ylabel!("Prob. of shared birthday based on data")

## TODO Do a test to show that this curve is higher than the uniform
# case.

## Use Julia's sampler
function data_birthday_simulation_julia(N::Int, data)
    w = fweights(data)
    birthdays = [sample(w) for i=1:N]
    # so each birthday is an integer from 1 to 365.
    # we need to check if all of the birthdays are unique
    return !allunique(birthdays) # ! reverses true -> false, false -> true
end
ntrials = 100
prob_dup_birthday = zeros(0) # this is an empty array
for people=5:40
    numdup = 0
    for t=1:ntrials
        numdup += data_birthday_simulation_julia(people, Int.(birthdays[:,2]))
    end
    push!(prob_dup_birthday, numdup/ntrials)
end
# make a quick people - by - prob table
display([5:40 prob_dup_birthday])
# but also make a plot
using Plots
plot(5:40, prob_dup_birthday)
xlabel!("People")
ylabel!("Prob. of shared birthday based on data")


"""
    lis_length(str_1::String, str_2::String)

The longest increasing subsequence (LIS) algorithm is dedicated to finding the longest
increasing subsequence for a given array.  This subsequence does not have to be contiguous
or unique.

    1. Figure out the number of possible different subsequences of a string with length `m` and `n`.
    2. Recontracted `for-loops` for defining the outer layer of the storing matrix `C`.
    3. Checking if `str_1[i] == str_2[j]` and updating the count in  matrix `C`.
    4.Return the latest element of `C[m,n]`.

For more information see: [https://en.wikipedia.org/wiki/Longest_increasing_subsequence](https://en.wikipedia.org/wiki/Longest_increasing_subsequence)

# Arguments
- `array::Array{Int64,1}`: array of integers unsortetd or sorted

# Examples
```julia-repl
julia> import ClassicAlgorithmsCollections
julia> arr = [10, 22, 9, 33, 21, 50, 41, 60]
julia> ClassicAlgorithmsCollections.lis_length(arr)
4
```
"""
function lis_length(array::Array{Int64,1})
    n = length(array)
    list = fill(1, n)
    for i in 2:n
        for j in 1:i
            if array[i] > array[j] && list[i] < list[j] + 1
                list[i] = list[j] + 1
            end
        end
    end

    result = 0
    for i in 1:n
        if list[i] > result
            result = list[i]
        end
    end
    return result
end


"""
Sieve of Eratosthenes is an algorithm for finding all the primes upto a limit `n`.

Reference:
-https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes
"""
function eratosthenes(n)
    primes = fill(true,n)
    primes[1] = false
    for p = 2:n
        primes[p] || continue
        for i = 2:div(n,p)
            primes[p*i] = false
        end
    end
    findall(primes)
end


## Before we got started
# I added the Images package via
# typing "]" then "add Images"
# alternatively, we could use
# using Pkg; Pkg.add("Images")

## Let's try and make Wikipedia's Julia set figure.
using Images
using Plots

## It says the seed coordinates are:
seeds = (-0.512511498387847167, 0.521295573094847167)

## Let's work with the Julia set.
# What we need to do is to check if, when starting
# from the complex number z=0, and iterating the map
# f(z) = z^2 + c
# for a large number of times (say 256)
# if the value of abs(f(z)) ever goes above 2,
# then we stop when that happens and return the value.
c = 0.25
# if we start at z = 0, then we test if z=0 is in the set.
z = 0.0+0.0im # this is how to make an imaginary number
f(z) = z^2 + c
@show f(z)
@show f(f(z))
@show f(f(f(z)))

##
zi = z
for i=1:10
    global zi
    zi = f(zi)
    @show zi
end
##
"""
`julia_check` is a subroutine useful for working with
the Julia set. We are going take as input two values:
z0 and c. The goal is to test whether or not the value
z0 seems to be in the Julia set for
    f(z) = z^2 + c
by starting from z0 = the input z0 and iterating
z1 = f(z0)
z2 = f(z1)
...
and so on.
Now, if at any point, we find abs(z) >= 2, then we know
that the point is NOT in the Julia set. In which case,
we return the iteration number where we detect this.
We run this for maxiter steps.
"""
function julia_check(z0, c; maxiter::Int=256)
    z = z0
    for i=1:maxiter
        z = z^2 + c
        if abs(z) >= 2
            return i
        end
    end
    return maxiter
end
julia_check(-1, 0.25)

## Let's look at starting positions z0 that range
## from [-1.75 to 1.75] in the real component
## and [-1.75 to 1.75] in the complex component
npts = 10
Z = zeros(npoints,npoints) # this will represent our Images
real_values = range(start=-1.75,stop=1.75,length=npts)
complex_values = range(start=-1.75,stop=1.75,length=npts)
for i=1:npts
    for j=1:npts
        z0 = real_values[i] + complex_values[j]*1.0im
        Z[j,i] = julia_check(z0, 0.25)
    end
end
heatmap(Z)

## The fix!
npts = 10
Z = zeros(npts,npts) # this will represent our Images
real_values = range(-1.75,stop=1.75,length=npts)
complex_values = range(-1.75,stop=1.75,length=npts)
for i=1:npts
    for j=1:npts
        z0 = real_values[i] + complex_values[j]*1.0im
        Z[j,i] = julia_check(z0, 0.25)
    end
end
heatmap(Z)

## Let's turn this into a function!
function julia_set(real_values, complex_values)
    Z = zeros(length(complex_values), length(real_values))
    for i=1:length(real_values)
        for j=1:length(complex_values)
            z0 = real_values[i] + complex_values[j]*1.0im
            Z[j,i] = julia_check(z0, 0.25)
        end
    end
    return Z
end
npts = 500
real_values = range(-1.75,stop=1.75,length=npts)
complex_values = range(-1.75,stop=1.75,length=npts)
heatmap(julia_set(real_values, complex_values))
##
function julia_set(real_values, complex_values; c=0.25)
    Z = zeros(length(complex_values), length(real_values))
    for i=1:length(real_values)
        for j=1:length(complex_values)
            z0 = real_values[i] + complex_values[j]*1.0im
            Z[j,i] = julia_check(z0, c)
        end
    end
    return Z
end
npts = 500
real_values = range(-1.75,stop=1.75,length=npts)
complex_values = range(-1.75,stop=1.75,length=npts)
heatmap(julia_set(real_values, complex_values; c = seeds[1] + seeds[2]*1.0im))
## Let's increase the resolution.
npts = 1000
real_values = range(-1.75,stop=1.75,length=npts)
complex_values = range(-1.75,stop=1.75,length=npts)
Z = julia_set(real_values, complex_values; c = seeds[1] + seeds[2]*1.0im)
## And then try and make our plot.
heatmap(log10.(Z))
## TODO or IDEA - Play around the with the colors to see if you find something you like better
## Show this like an image.
image = Gray.(Z/maximum(Z))
image

## to save this picture, we can use
using FileIO
using ImageMagick
save("juliaset.png", Gray.(Z/maximum(Z)))


##########################################################
#FEniCS tutorial demo program: Poisson equation with Dirichlet conditions.
#Test problem is chosen to give an exact solution at all nodes of the mesh.
#  -Laplace(u) = f    in the unit square
#            u = u_D  on the boundary
#  u_D = 1 + x^2 + 2y^2
#    f = -6
###########################################################

module ft01

using FenicsPy

# Create mesh and define function space
mesh = UnitSquareMesh(8, 8)
V = FunctionSpace(mesh, "P", 1)

# Define boundary condition
u_D = Expression("1 + x[0]*x[0] + 2*x[1]*x[1]", degree=2)

bc = DirichletBC(V, u_D, "on_boundary")

# Define variational problem
u = TrialFunction(V)
v = TestFunction(V)
f = Constant(-6.0)
a = dot(grad(u), grad(v))*dx
L = f*v*dx

# Compute solution
u = FeFunction(V)
solve(a == L, u, bc)

# Save solution to file in VTK format
vtkfile = File("poisson/solution.pvd")
vtkfile << u

# Compute error in L2 norm
error_L2 = errornorm(u_D, u, "L2")

# Compute maximum error at vertices
vertex_values_u_D = u_D.compute_vertex_values(mesh)
vertex_values_u = u.compute_vertex_values(mesh)
error_max = max(abs.(vertex_values_u_D - vertex_values_u)...)

# Print errors
println("error_L2  =", error_L2)
println("error_max =", error_max)

# Plot solution and mesh
#plot(u)
#plot(mesh)

end # module ft01


using FractalTools
using Plots

function example_attractor_of_ifs(N_iter)
    # An example of one dimensional ifs
    A1 = reshape([1/2],1,1)
    b1 = [0]
    A2 = reshape([1/2],1,1)
    b2 = [1/2]

    w1 = Transformation(A1,b1)
    w2 = Transformation(A2,b2)

    w = [w1, w2]
    ifs = IFS(w)
    initset = [[0.1]]
    atr = attractor(ifs, initset; alg=RandAlg(), numiter=N_iter,allocated=true)
    attrset = vcat(atr.set...)
    t = ones(length(attrset))
    return atr, attrset, t
end

atr, attrset, t = example_attractor_of_ifs(10)
p1 = scatter(t, attrset, title="Attractor of the IFS", label="Attractor Points", markersize = 4)

atr, attrset, t = example_attractor_of_ifs(100)
p2 = scatter(t, attrset, title="Attractor of the IFS", label="Attractor Points", markersize = 4)

atr, attrset, t = example_attractor_of_ifs(10000)
p3 = scatter(t, attrset, title="Attractor of the IFS", label="Attractor Points", markersize = 4)

plot(p1, p2, p3, layout=(1,3))
xlabel!("x")
ylabel!("z")
xlims!((0.9,1.1))
# savefig("myplot.png")

# An example of atr with Channel
atr = attractor(ifs1, initset; alg=RandAlg(), numiter=100, allocated=false)
collect(atr.set)
take!(atr.generator)
# An example of atr with randalg_sequential_generator
# generator = randalg_sequential_generator(ifs1.ws, ifs1.probs)
# take!(generator)

# An example of two dimensional ifs
A = [1/2 0;0 1/2]
b1 = [0; 0]
b2 = [0; 1/2]
b3 = [1/2; 1/2]
b4 = [1/2; 0]

w1 = Transformation(A,b1)
w2 = Transformation(A,b2)
w3 = Transformation(A,b3)
w4 = Transformation(A,b4)

w = [w1, w2, w3, w4]
ifs2 = IFS(w)
ifs2.ws
ifs2.probs

# An example of different probabilities

p = [0.2, 0.4, 0.1, 0.3]

ifs3 = IFS(w,p)
ifs3.ws
ifs3.probs


# Root finder functions for 1d func
# Newton's Method, Linear Approximation
# Note that you can also find root by applying optimizer on an Error function
# ϵ(x,...)

# Newton's Method
# f1 is δf╱δx
function solve_Newton(f,f1,par;nlim = 1e4,ϵ=1e-3)
    for i in 1:nlim
        par1 = par - f(par)/f1(par)
        if ϵ>abs(par1 -par)
            return(par1)
        end
        par=par1
    end
    println("Warning! unconverged after" * string(nlim) *"Iterations.")
    return(par1)
end

# Linear Approximation
function solve_Linear_Approx(f,par1,par2;nlim = 1e4,ϵ=1e-3)
    #must follow :x1 * x2<0
    #             par1<par2
    for i in 1:nlim
        x1 = f(par1)
        x2 = f(par2)
        global par3 = par1+(par2 - par1)*abs(x1/(x1-x2))
        if abs(par3-par1)<ϵ
            if abs(par3 - par2)<ϵ
                return par3
            end
        else
            if (par3*par1)>0
                par1=par3
            else (par3*par2)>0
                par2=par3
            end
        end
    end
    println("Warning! unconverged after"*string(nlim)*"iterations.")
    return par3
end

#----------------------------------------------------------------
using Plots
x = 1:.1:4
y = sin.(x)
plot(x,y)
plot!(x,zeros(length(x)))

f(x) = sin(x)
f1(x) = cos(x)

solve_Newton(f,f1,3)

solve_Linear_Approx(f,3,4)
#----------------------------------------------------------------


"""
    ceil_val(x)

Finds the ceiling of x as an Integer

# Example
```julia
ceil_val(1.3)   # 2.0
ceil_val(2.0)   # returns 2.0
ceil_val(-1.5)  #returns -1.0
````

# Reference
- https://en.wikipedia.org/wiki/Floor_and_ceiling_functions

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function ceil_val(x)
    return x -trunc(x) <= 0 ? trunc(x) : trunc(x)+1
end

"""
    floor_val(x)

Finds the floor of x as an Integer

# Example
```julia
floor_val(1.3)  # 1
floor_val(2.0)  # returns 2.0
floor_val(-1.7) # returns -2.0
````

# Reference
- https://en.wikipedia.org/wiki/Floor_and_ceiling_functions

Contributed By:- [Ashwani Rathee](https://github.com/ashwani-rathee)
"""
function floor_val(x)
    return x -trunc(x) >= 0 ? trunc(x) : trunc(x)-1
end


# Number of iterations Euclidean Algorithm needs for gcd(a, b)
function gcd_iters(a, b)
    iters = 0
    while b != 0
        iters += 1
        a, b = b, a % b
    end
    return iters
end

# Number of bits needed to represent some non-negative n
function num_bits(n)
    ct = 1
    n >>= 1
    while n != 0
        n >>= 1
        ct += 1
    end
    return ct
end

# Test num_bits
num_bits_tests = [(0, 1), (1, 1), (2, 2), (3, 2), (4, 3), (5, 3), (6, 3), (7, 3), (8, 4)]
for (num, ans) in num_bits_tests
    @assert num_bits(num) == ans
end

small_numbers = collect(0:2^12-1)
fibs = [0, 1] # Fibonacci numbers
for i in 1:90
    push!(fibs, fibs[length(fibs) - 1] + fibs[length(fibs)])
end
two_powers = [2 ^ x for x in 1 : 60]
mersenne_numbers = [x - 1 for x in two_powers]

operands_of_interest = Set(vcat(small_numbers, fibs, two_powers, mersenne_numbers))

# Verify that for all operands of interest, num iters is <= 1 + num_bits(y) ÷ log2_of_φ (note floor divison)
log2_of_φ = log2(MathConstants.golden)
for x in operands_of_interest, y in operands_of_interest
# for x in 0:2^16-1, y in 0:2^16-1
    iters = gcd_iters(x, y)
    if 1 + num_bits(y) ÷ log2_of_φ < iters
        println("gcd($(x), $(y)) takes $(iters) iters (max bits: $(max_bits), ratio: $(iters / max_bits))")
    end
    @assert iters <= 1 + num_bits(y) ÷ log2_of_φ
end
println("Done")

#==
Operands where (# of iterations)/(max_bits) = 1.5 (none have a higher ratio, up through 14 bits)
Note that all are sequential fibonacci #s
gcd(2, 3) takes 3 iters (max bits: 2)
gcd(8, 13) takes 6 iters (max bits: 4)
gcd(34, 55) takes 9 iters (max bits: 6)
gcd(144, 233) takes 12 iters (max bits: 8)
gcd(610, 987) takes 15 iters (max bits: 10)

https://stackoverflow.com/questions/9060816/running-time-of-euclids-gcd-algorithm
y = second arg
b = # of bits in second arg
bound is:
1 + log(φ, y)
1 + log(φ, 2^b - 1)
1 + log(2, 2^b - 1)/log(2, φ)
1 + b/log(2, φ)

experimentally verified for all numbers representable in 16 bits

==#


struct BracketingInterval
    x_lo
    x_mid
    x_hi
    f_lo
    f_mid
    f_hi
    iter
    success::Bool
end

function Base.show(io::IO, bracket::BracketingInterval)
    if bracket.success
        print("Successful seach")
    else
        print("Failed seach")
    end
    print(" with $(bracket.iter)-iterations\n")
    println("\tx\tf(x)")
    println( "lo:\t$(bracket.x_lo)\t$(bracket.f_lo)" )
    println( "mid:\t$(bracket.x_mid)\t$(bracket.f_mid)" )
    println( "hi:\t$(bracket.x_hi)\t$(bracket.f_hi)" )
end

function bracket_min(
    fcn,
    x_lo,
    x_hi;
    step = abs(x_hi-x_lo) * 1e-2,
    step_factor = 1.61,
    max_iter = 1000)

    f_x_lo = fcn(x_lo)
    f_x_hi = fcn(x_hi)

    if f_x_lo < f_x_hi
        # swap the orders
        x_lo, x_hi = x_hi, x_lo
        f_x_lo, f_x_hi = f_x_hi, f_x_lo
        step = -step
    end

    iter = 1
    while iter <= max_iter
        x_next = x_hi + step
        f_next = fcn(x_next)
        if f_next > f_x_hi
            # bracketed interval found
            return BracketingInterval(x_lo, x_hi, x_next, f_x_lo, f_x_hi, f_next, iter, true)
        end

        x_lo, x_hi = x_hi, x_next
        f_x_lo, f_x_hi = f_x_hi, f_next

        step *= step_factor
        iter += 1
    end
    println("Failed to find a bracketing interval in the prescribed number of iterations")
    return BracketingInterval(x_lo, x_hi, x_next, f_x_lo, f_x_hi, f_next, iter, false)

end

struct GoldenSectionSearchStatistics
    x_sol
    y_sol
    iter
end

function golden_section_search(fcn, x_lo, x_hi; max_iter = 100, x_tol = 1e-10)

    rho = 0.5 * (sqrt(5)-1)
    if x_lo > x_hi
        # swap
        x_lo, x_hi = x_hi, x_lo
    end
    a = x_lo
    b = x_hi

    c_fcn(lo, hi) = rho * lo + (1 - rho) * hi
    d_fcn(lo, hi) = (1 - rho) * lo + rho * hi

    y_a = fcn(a)
    y_b = fcn(b)

    c = c_fcn(a, b)
    d = d_fcn(a, b)

    y_c = fcn(c)
    y_d = fcn(d)

    x_sol = NaN
    y_sol = NaN
    iter = 0
    for outer iter = 1:max_iter

        if y_c < y_d
            # choose (a, c, d)
            a, d, b = a, c, d
            y_a, y_d, y_b = y_a, y_c, y_d
            c = c_fcn(a, b)
            y_c = fcn(c)
        else
            # choose (c, d, b)
            a, c, b = c, d, b
            y_a, y_c, y_b = y_c, y_d, y_b
            d = d_fcn(a, b)
            y_d = fcn(d)
        end

        if abs(b - a) <= x_tol
            (x_sol, y_sol) = (y_c < y_d) ? (c, y_c) : (d, y_d)
            break
        end

    end
    if iter == max_iter
        (x_sol, y_sol) = (y_c < y_d) ? (c, y_c) : (d, y_d)
    end

    return GoldenSectionSearchStatistics(x_sol, y_sol, iter)

end


# p12.jl - accuracy of Chebyshev spectral differentiation
#         (compare p7.jl)

# Compute derivatives for various values of N:
Nmax = 50; allN = 6:2:Nmax; E = zeros(4,Nmax);
for N = 1:Nmax
    (D,x) = cheb(N);
    v = @. abs(x)^3; vprime = @. 3*x*abs(x);   # 3rd deriv in BV
    E[1,N] = norm(D*v-vprime,Inf);
    v = @. exp(-x^(-2)); vprime = @. 2*v/x^3;  # C-infinity
    E[2,N] = norm(D*v-vprime,Inf);
    v = @. 1/(1+x^2); vprime = @. -2*x*v^2;    # analytic in [-1,1]
    E[3,N] = norm(D*v-vprime,Inf);
    v = x.^10; vprime = 10*x.^9;               # polynomial
    E[4,N] = norm(D*v-vprime,Inf);
end

# Plot results:
titles = [L"|x|^3",L"\exp(-x^2)",L"1/(1+x^2)",L"x^{10}"];  clf();
for iplot = 1:4
    subplot(2,2,iplot);
    semilogy(1:Nmax,E[iplot,:],".-",markersize=6);
    axis([0,Nmax,1e-16,1e3]); grid(true);
    xticks(0:10:Nmax); yticks(10.0.^(-15:5:0));
    title(titles[iplot]);
    iplot > 2 ? xlabel("N") : nothing;
    iplot%2 > 0? ylabel("error") : nothing;
end


#' ## Run this cell to load the graphics packages
#+ results="hidden"
using Plots
#gr()
#plotly()
pyplot()
#+

#' ## Adding a function parameter
#' In the last notebook, we saw an example of adding **parameters** to
#' functions. These are values that control the behavior of a function.
#' Let's look at that in some more detail.
#' Let's go back to our original version of the σ function:
#+ results="hidden"
sigma(x) = 1/(1+exp(-x))
#+
#' Instead of working with a single function, we can work with a whole class
#' (set) of functions that look similar but differ in the value of a
#' **parameter**. Let's make a new function that uses the previous $\sigma$
#' function, but also has a parameter, $w$. Mathematically, we could write

#' $$f_w(x) = f(x; w) = \sigma(w \, x).$$

#' (Here, $w$ and $x$ are multiplied in the argument of $\sigma$; we could
#' write $w \times x$ or $w * x$, or even $w \cdot x$, but usually the symbols
#' are not written.)

#' Mathematically speaking, we can think of $f_w$ as a different function for
#' each different value of the parameter $w$.

#' In Julia, we write this as follows:
#+ results="hidden"
f(x, w) = sigma(w * x)
#+
#' Note that Julia just treats parameters as additional *arguments* to the
#' function; the function `f` has two arguments, the value of `x` and the
#' value of `w` that we want to use.

#' We can now investigate the effect of $w$ interactively. To do so, we need
#' a way of writing in Julia "the function of one variable $x$ that we obtain
#' when we fix the value of $w$". We write this as an "anonymous function", as
#' we saw in the notebook on functions:
#'
#'     x -> f(x, w)
#'
#' We can read this as "the function that maps $x$ to the value of $f(x, w)$,
#' for a value of $w$ that was previously given".

#' Now we are ready to draw the function. For each plot, we *fix* a value of
#' the parameter $w$ and draw the resulting function as a function of $x$.
#' We can interactively modify the value of $w$, and plot the new function that
#' comes out:
#' ## Try manipulating `w` between -2 and 2
w = -1
plot(x->f(x, w), -5, 5, ylims=(0,1), label="sigmoid")
plot!(x->(x>0), -5,5, label="Square Wave")

#' #### Exercise 1
#'
#' Try writing your own function that takes a parameter. Start by copying and
#' executing
#'
square(x) = x^2
#'
#' Then use `square` to declare a new function `square_and_scale` that takes
#' two inputs, `a` and `x` such that
#'
#' $$\mathrm{square\_and\_scale}(x; a) := a \cdot x^2$$

#' ####
function square_and_scale(x, a)
    return a*x^2
end

#' We can write some tests as follows, using the approximate equality operator,
# `≈`, written as `\approx<TAB>`:
square_and_scale(1, 1) ≈ 1
square_and_scale(1, 2) ≈ 2
square_and_scale(2, 1) ≈ 4
square_and_scale(2, 2) ≈ 8

# #### Exercise 2

#' Once you have declared `square_and_scale`, uncomment the code below and see
#' how the parameter `a` scales the function `square` :

x = -10:10
a = 5.0 # Try manipulating a here

plot(x, square.(x), label="x^2")
plot!(x, square_and_scale.(x, a), ls=:dash, label="ax^2")

#' ## Fitting a function to data
#' As we saw in the previous notebook, what we would like to do is use the
#' fact that we now have a parameter in our function in order to do something
#' useful! Namely, we want to model data with it.

#' So suppose that we are given a single data point $(x_0, y_0) = (2, 0.8)$.
#' We can try to "fit" the function $f_w$ by adjusting the parameter $w$ until
#' the function passes through the data.
#'
#' **Game**: Change `w` until the graph of the function hits the data point.
#' Which value of $w$ does that correspond to?
x0, y0 = 2, 0.8

#' f(x, w) = sigma(w * x)
w = 0.7 # try mainipulatoing `w` between -2 and 2
plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f")
scatter!([x0], [y0], label="data")
#' ## Quantifying how far we are from the goal: the *loss function*
#' We can see visually when the graph of the function passes through the data
#' point. But the goal is to be able to automate this process so that the
#' computer can do it unaided.
#'
#' So we will need a more precise way of deciding and quantifying
#' (i.e. measuring with a number) *how far away we are from the goal*;
#' here, the goal means hitting the data point with the function.

#' #### Exercise 3
#' Can you think of a way of measuring how far away our function is from
#' the data point?

#' ### Defining the loss function
#' We need to measure how far away the curve is from the data point when we
#' choose a particular value of $w$.
#' One way to do this is by finding the vertical distance $d$ from the curve
#' to the data point.
#'
#' Instead of just taking the distance, it is common to take the *square* of
#' the distance, $d^2$.
#'
#' Since we are taking the vertical distance, we need the distance at the given
#' value of $x_0$ where the data point lies. For a given value of the parameter
#' $w$, the height of the point on the curve with that value of $x_0$ is
#' $f(x_0, w)$.
#'
#' So we take
#'
#' $$d := y_0 - f(x_0, w)$$
#'
#' and
#'
#' $$d^2 = [y_0 - f(x_0, w)]^2.$$
#'
#' This is our measure of distance. It will change when $w$ changes -- in other
#' words, it is itself a *function of $w$*; we will denote this function by
#' $L(w)$, and call it the **loss function**:
#'
#' $$L(w) := [y_0 - f(x_0, w)]^2.$$
#'
#' So the goal is to find the value $w^*$ of $w$ where the loss function is *least*; in other words, we need to *minimize* the loss function!
#'
#' (Another name for a loss function is a *cost function*.)

#' #### Exercise 4
#'
#' (a) Define the loss function `L(w)` in Julia.
#+ results="hidden"
x0, y0 = 2, 0.8
L(w) = (y0 - f(x0,w))^2
#+

#' (b) Draw the data point and the function `x -> f(x, w)`.
#' Also draw a vertical line from the data point to the function `x -> f(x, w)`.
w = 0.5
pl = plot(x->f(x,w), -5, 5, ylims=(0,1), label="f_w", lw=3)
scatter!(pl, [x0], [y0], label="data")
plot!([x0, x0], [y0, f(x0, w)], label="loss", legend=(0.2,0.9))
#' (c) Make the plot interactive.
#'
#' (d) Add as the plot title the value of the loss function for the current value
#' of $w$.
title!("L(w) = $(round(L(w); sigdigits = 5))")
#'
#' (e) Use the slider to find the value $w^*$ of $w$ for which the loss function
#' reaches its minimum value. What is $w^*$? What is the value of the loss
#' function there, $L(w^*)$?
w = 0.7
pl = plot(x->f(x,w), -5, 5, ylims=(0,1), label="f_w", lw=3)
scatter!(pl, [x0], [y0], label="data")
plot!([x0, x0], [y0, f(x0, w)], label="loss", legend=(0.1,0.9))
title!("L(w) = $(round(L(w); sigdigits = 5))")

#' ## What does the loss function look like?
#' The loss function $L(w)$ tells us how far away the function $f_w$ is from
#' the data when the parameter value is $w$, represented visually as the
#' vertical line in the previous plot. When the data are fixed, this is a
#' function only of the parameter $w$. What does this function look like as
#' a function of $w$? Let's draw it!

#' #### Exercise 5
#' Draw the function $L(w)$ as a function of $w$.

#' #### Solution
pl = plot(L, -2, 2, xlabel="w", ylabel="L(w)", ylims=(0, 0.7), label="L", legend=(0.8,0.9))
#' This graph quantifies how far we are from the data point for a given value
#' of $w$.
#' What features can we see from the graph?
#'
#' Firstly, we see that $L(w)$ is always bigger than $0$, for any value of $w$.
#' This is because we want $L$ to be some kind of measure of *distance*, and
#' distances cannot be negative.
#'
#' Secondly, we see that there is a special value $w^*$ of $w$ where the
#' function $L$ reaches its minimum value. In this particular case, it actually
#' reaches all the way down to $0$!
#' This means that the original function $f$ (the one we manipulated above)
#' passes exactly through the data point $(x_0, y_0)$.

#' #### Exercise 6
#' Draw a zoomed-in version of the graph to find the place $w^*$ where the
#' function hits $0$ more precisely.
pl = plot(L, 0.5, 1, xlabel="w", ylabel="L(w)", ylims=(0, 0.01), label="L", legend=(0.8,0.9))

#' ### A different way of defining the loss function
#' **Why did we use such a complicated function $L$ with those squares inside?**
#' We could instead just have used the absolute distance, instead of the
#' distance squared, using the *absolute value* function, written mathematically
#' as $| \cdot |$, and in Julia as `abs`.

#' #### Exercise 7
#'
#' Define a new loss function, `L_abs`, using the absolute value, and see what
#' it looks like.

#' #### Solution
L_abs(w) = abs(y0 - f(x0,w))

pl = plot(L_abs, 0.5, 1, xlabel="w", ylabel="L_abs(w)", ylims=(0, 0.01), label="L", legend=(0.8,0.9))

#' Now we see why it was previously generally preferred to use squares: using
#' the absolute value gives a cost function that is *not smooth*. This makes it
#' difficult to use methods from calculus to find the minimum. Nonetheless,
#' using non-smooth functions is very common in machine learning nowadays.


"""
    conGrad(A,B,Xawal)
adalah fungsi yang digunakan untuk mencari solusi SPL `Ax=B` secara iteratif dengan
nilai tebakan awal `Xawal` menggunakan metode conjugate gradient.

# Example
```jl
julia> A = [2  3  3
        3  5  4
        3  4  6];

julia> B = [3,4,6];

julia> X,flag,err,M = conGrad(A,B,[0 0 0.9]);

julia> X
3-element Array{Float64,1}:
  2.380760976839332e-6
 -1.220333023839802e-6
  0.9999996142293243

julia> flag
0

julia> err
8.019009497939746e-8

julia> M
229×5 Array{Float64,2}:
   0.0  0.0          0.0          0.9       NaN
   1.0  0.0266764    0.0355685    0.953353    0.0694497
   2.0  0.0154769    0.000266424  0.975767    0.0432903
   3.0  0.0193622    0.00858077   0.976843    0.00924021
   ⋮
 225.0  5.10651e-6  -1.83459e-6   0.999999    1.32762e-6
 226.0  2.29335e-6  -1.29137e-6   1.0         3.08384e-6
 227.0  2.4522e-6   -1.23209e-6   1.0         1.73709e-7
 228.0  2.38076e-6  -1.22033e-6   1.0         8.01901e-8
```
return solusi `X` dengan `flag` bernilai 0 jika metode conjugate gradient
berhasil menemukan solusi dan gagal jika tidak 0.
Serta, matriks `M` yang berisi catatan proses tiap iterasi `[k, X', error]`
"""
function conGrad(A,B,Xawal)
    if ~(isposdef(A) && issymmetric(A))
        error("matriks A harus simetrik definit positif")
    end
    delta = 10^-7;
    maxi = 1000;
    flag = 1;
    X = Xawal[:];
    r = B-A*X
    d = r
    M = [0 X' NaN];
    for k = 1:maxi
        Xlama = X
        rlama = r
        dlama = d
        X = Xlama + ((rlama'rlama)/(dlama'A*dlama))*dlama
        r = rlama - ((rlama'rlama)/(dlama'A*dlama))*A*dlama
        d = r - ((r'r)/(rlama'rlama))*dlama
        err = norm(X-Xlama)
        M = [M; [k X' err] ];
        if err<delta
            flag = 0;
            break
        end
    end
    err = M[end,end]
    return X, flag, err, M
end


function solve_euler(timestep::Float64, n::Int64)
    euler_result = Vector{Float64}(undef, n)

    # Setting the initial condition
    euler_result[1] = 1;
    for i = 2:length(euler_result)
        euler_result[i] = euler_result[i-1] - 3.0*euler_result[i-1]*timestep
    end
    return euler_result
end

function check_result(euler_result::Vector{Float64}, threshold::Float64,
                      timestep::Float64)
    is_approx = true

    for i = 1:length(euler_result)
        time = (i - 1)*timestep
        solution = exp(-3*time);
        if (abs(euler_result[i] - solution) > threshold)
            println(euler_result[i], solution)
            is_approx = false
        end
    end

    return is_approx
end

function main()
    timestep = 0.01
    n = 100
    threshold = 0.01

    euler_result = solve_euler(timestep,n)
    is_approx = check_result(euler_result, threshold, timestep)

    println(is_approx)
end

main()