"""
Julia program to find the Delannoy number for a given rectangular grid.
A Delannoy number describes the number of paths from the southwest corner
(0, 0) of a rectangular grid to the northeast corner (m, n) using only single steps north, northeast, or east.
Delannoy number is named after the French mathematican Henri Delannoy.
"""

function delannoy_number(r, c)

    # Initialize the dp array with '0' as value.
    dp = zeros(Int, c + 1, r + 1)

    for i in 1:c
        dp[1,i] = 1
    end
    for i in 2:c+1
        dp[i,1] = 1
    end
    # From Each point calculate the number of paths, to the north-east point, that can be reached by
    # traversing through the immediate right or immediate top or immediate top-right point.
    for i in 2:c+1
        for j in 2:r+1
            dp[i,j] = dp[i-1,j] + dp[i-1,j-1] + dp[i,j-1]
        end
    end
    return dp[c+1,r+1]
end

print("Enter the co-ordinates of the north-east corner.\nEnter the row co-ordinate: ")
x = readline()
x = parse(Int, x)
print("Enter the column co-ordinate: ")
y = readline()
y = parse(Int, y)
res = delannoy_number(x, y)
print("The Delannoy number of the given grid is $res.")

"""
Time Complexity - O(x * y), where `x`, `y` is the given co-ordinates.
Space Complexity - O(1)

SAMPLE INPUT AND OUTPUT

SAMPLE I

Enter the co-ordinates of the north-east corner.
Enter the row co-ordinate: 5
Enter the column co-ordinate: 6
The Delannoy number of the given grid is 3653.
"""


@doc raw"""
    triag_rng(a, b, m, shape=1; seed=nothing)

Generate a `shape` element array of random variables from a Triangular(`a`, `b`, `m`) distribution. Optionally you can set a specific seed.

# Notes

The Triangular distribution is given by:

```math
f(x, a, b, m) = \begin{cases}
\frac{2(x-a)}{(m-a)(b-a)}  & \text{if  } a < x ≤ m, \\
\frac{2(b-x)}{(b-m)(b-a)}  & \text{if  } m < x ≤ c, \\
0 & \text{otherwise}
\end{cases}
```

# Examples

```julia-repl
julia> triag_rng()
1-element Vector{Float64}:
 0.3559088458688944

julia> triag_rng(0,7,2,5)
5-element Vector{Float64}:
 1.3758072115332673
 6.049452463477447
 6.042781317411027
 2.914959243260448
 4.454707036522528
```

# References

W. Stein and M. Keblis. A new method to simulate the triangular distribution. Mathematical and Computer Modelling, Volume 49, Issues 5–6, 2009.
"""
function triag_rng(a=0,b=1,m=.5,shape=1)
    u = get_std_uniform(shape)
    v = get_std_uniform(shape)
    t_min = min.(u,v)
    t_max = max.(u,v)
    c = (m-a)/(b-a)
    t = ((1-c) .* t_min) .+ (c .* t_max)
    X = a .+ (b-a) .*t
    return X
end


using Random, Distributions, LinearAlgebra
using Plots; pyplot()

function sum_of_squares(v)
    """Computes the sum of squared elements in v"""
    return dot(v, v)
end

function difference_quotient(f, x, h)
    return (f(x + h) - f(x)) / h
end

function square(x)
    return x * x
end

function derivative(x)
    return 2 * x
end

@assert 4 <= difference_quotient(square, 2, .001) <=4.01

function squared_distance(v, w)
    """Computes (v_1 - w_1) ** 2 + ... + (v_n - w_n) ** 2"""
    return sum_of_squares((v - w))
end

function distance(v, w)
    """Computes the distance between v and w"""
    return sqrt(squared_distance(v, w))
end

function partial_difference_quotient(f, v, i, h)
        """Returns the i-th partial difference quotient of f at v"""
        # w = [v_j + (h if j == i else 0) for j, v_j in enumerate(v)]
        w = copy(v)
        for j in 1:length(v)
            v_j = v[j]
            if(j==i)
                w[j] = v_j + h
            else
                w[j] = v_j
            end
        end

        return (f(w) - f(v)) / h
end

function add(v, w)
    """Adds corresponding elements"""
    @assert length(v) == length(w)
    return v+w
end

function scalar_multiply(c, v)
    """Multiplies every element by c"""
    return [c * v_i for v_i in v]
end

function gradient_step(v, gradient, step_size)
    """Moves `step_size` in the `gradient` direction from `v`"""
    @assert length(v) == length(gradient)
    step = scalar_multiply(step_size, gradient)
    return add(v, step)
end

function sum_of_squares_gradient(v)
    return [2 * v_i for v_i in v]
end

function custom_fun(v)
    sign = -1
    if(v[2] > 4)
        sign = 1
    end
    return v[1]*v[1] + sign * v[2]*v[2] + 16
end

function estimate_gradient(f, v, h = 0.0001)
    return [partial_difference_quotient(f, v, i, h) for i in 1:length(v)]
end
# let v = [rand(-10: 10) for i in 0:2]
let v = [2.0, 2.0]
    for epoch in 1:1000
        # grad = sum_of_squares_gradient(v)    # compute the gradient at v
        grad = estimate_gradient( custom_fun,v)
        v = gradient_step(v, grad, -0.001)    # take a negative gradient step
    end

    # @assert distance(v, [0, 0]) < 0.001    # v should be close to 0
    @assert custom_fun(v) < .1
end

xs = -10: 11
actuals = [derivative(x) for x in xs]
estimates = [difference_quotient(square, x, 0.001) for x in xs]

# plot to show they're basically the same
plot(title = "Actual Derivatives vs. Estimates")
plot!(xs, actuals, label="Actual", seriestype = :scatter, color = "red", markersize = 7)       # red  x
plot!(xs, estimates, label="Estimate", seriestype = :scatter, color = "blue", markershape= :x)   # blue +
# plot(legend=loc=9)
# plot.show()

result = estimate_gradient(sum_of_squares,[2.0 2.0])
println(result) #should be equal to [4.0 4.0]

function error(v)
    x, y, slope, intercept = v
    predicted = slope * x + intercept
    error = (predicted - y)^2
    return error
end

println(error([2, 1 , 3, 5]))

ex_vector = [2.0, 1.0 , 2.0, 5.0]
println(partial_difference_quotient(error,ex_vector,3,.001))

function linear_gradient(x, y, theta)
    vector = [x,y, theta...]
    error_diff_estimate = [partial_difference_quotient(error,vector,i,.001)
                        for i in 3:4]

end
error_diff_estimate = linear_gradient(2.0,1.0, [2.0, 5.0])
println(error_diff_estimate)

inputs = [(x, 20 * x + 5) for x in -50: 49]

function vector_mean(vectors)
    """Computes the element-wise average"""
    n = length(vectors)
    return scalar_multiply(1/n, sum(vectors))
end

ud = Uniform(-1,1)
let theta = [rand(ud), rand(ud)]

learning_rate = 0.001
for epoch in 1:3000
    grad = vector_mean([linear_gradient(x,y, theta) for (x,y) in inputs])

    theta = gradient_step(theta, grad, -learning_rate)
    println(epoch, theta)
end

slope, intercept = theta
@assert 19.9 < theta[1] < 20.1
@assert 4.9 < intercept < 5.1
end

#finished


exp(π) - π

# Solve x^2 + 5x + 6 = 0 using the quadratic formula (real arithmetics)

# Coefficients a,b,c in ax^2 + bx + c = 0
a = 1
b = 5
c = 6

# The quadratic formula
d = sqrt(b^2 - 4*a*c)
r1 = (-b - d) / 2a
r2 = (-b + d) / 2a
println("The roots are ", r1, " and ", r2)

function myfunc(x,y)
    x + y
end

myfunc(1,2)

myfunc2(x,y) = x + 2y
myfunc2(3,5)

myfunc3 = (x,y) -> x + 3y
myfunc3(3,5)

# Same thing without giving the function a name:
((x,y) -> x + 3y)(3,5)

function mynewfunc(x,y)
    out1 = x + y
    out2 = out1 * (2x + y)
    out1, out2
end

y1, y2 = mynewfunc(2,1)

function real_roots_of_quadratic(a,b,c)
    # Compute the real roots of the quadratic ax^2 + bx + c = 0
    d = sqrt(b^2 - 4*a*c)
    r1 = (-b - d) / 2a
    r2 = (-b + d) / 2a
    r1, r2
end

real_roots_of_quadratic(1, 5, 6)

real_roots_of_quadratic(-1, 5, 6)


"""
    beta_rng(α, β, shape=1; seed=nothing)

Generate a `shape` element array of random variables from a Beta(`α`, `β`) distribution. Optionally you can set a specific seed.

# Notes

The Beta distribution is given:

``f(x,α,β) = \\frac{x^{α-1 (1-x)^{β-1}}}{Β(α,β)} \\quad x \\in [0,1]``

where ``Β(α,β) = \\frac{Γ(α)Γ(β)}{Γ(α+β)}``

# Examples

```julia-repl
julia> beta_rng(1,2)
1-element Vector{Float64}:
 0.44456674672905633

julia> beta_rng(1, 2, (2,2))
2×2 Matrix{Float64}:
 0.0792132  0.595657
 0.737615   0.649721

```

# References

D.P. Kroese, T. Taimre, Z.I. Botev. Handbook of Monte Carlo Methods.
  Wiley Series in Probability and Statistics, John Wiley & Sons, New York, 2011.

Law, A. Simulation modeling and analysis, 5th Ed. McGraw Hill Education, Tuscon, 2013.
"""
function beta_rng(α::Real, β::Real, shape::Union{Int, Tuple{Vararg{Int}}}=1; seed::Union{Int, Nothing}=nothing)
    seed_setter(seed)
    Y₁ = gamma_rng(α, 1, shape, seed=seed)
    Y₂ = gamma_rng(β, 1, shape, seed=seed)
    X = Y₁ ./ (Y₁ .+ Y₂)
    return X
end


#= Finding the Greatest Common Divisor of 2 numbers using the Euclidean
Formula to lessen the time complexity.
=#

## Function

function GCD(a, b)
    if (a == 0)
        return b
    end
    if (b == 0)
        return a
    end
    if (a < b)
        a, b = b, a
    end
    ans = a % b
    while (ans != 0)
        a = b
        b = ans
        ans = a % b
    end
    return b
end

## Input

a = readline()
a = parse(Int64, a)
b = readline()
b = parse(Int64, b)

## Calling the function

GCD(a, b)

#=
Sample Test case:
Input:
    a = 1001  b = 2352
Output:
    7
Time complexity: O( log (min(a,b)) )
=#


# # Huber regression

# This example can be found here: <https://web.stanford.edu/~boyd/papers/pdf/cvx_applications.pdf>.
# Here we set `big_example = false` to only generate a small example which takes less time to run.
big_example = false
if big_example
    n = 300
    number_tests = 50
else
    n = 100
    number_tests = 20
end

# Generate data for Huber regression.
using Random
Random.seed!(1);
number_samples = round(Int,1.5*n);
beta_true = 5*randn(n);
X = randn(n, number_samples);
Y = zeros(number_samples);
v = randn(number_samples);

#-

## Generate data for different values of p.
## Solve the resulting problems.
using Convex, SCS, Distributions
lsq_data = zeros(number_tests);
huber_data = zeros(number_tests);
prescient_data = zeros(number_tests);
p_vals = range(0, stop=0.15, length=number_tests);
for i=1:length(p_vals)
    p = p_vals[i];
    ## Generate the sign changes.
    factor = 2 * rand(Binomial(1, 1-p), number_samples) .- 1;
    Y = factor .* X' * beta_true + v;

    ## Form and solve a standard regression problem.
    beta = Variable(n);
    fit = norm(beta - beta_true) / norm(beta_true);
    cost = norm(X' * beta - Y);
    prob = minimize(cost);
    solve!(prob, SCS.Optimizer(verbose=0));
    lsq_data[i] = evaluate(fit);

    ## Form and solve a prescient regression problem,
    ## i.e., where the sign changes are known.
    cost = norm(factor .* (X'*beta) - Y);
    solve!(minimize(cost), SCS.Optimizer(verbose=0))
    prescient_data[i] = evaluate(fit);

    ## Form and solve the Huber regression problem.
    cost = sum(huber(X' * beta - Y, 1));
    solve!(minimize(cost), SCS.Optimizer(verbose=0))
    huber_data[i] = evaluate(fit);
end

#-
using Plots

plot(p_vals, huber_data, label="Huber", xlabel="p", ylabel="Fit" )
plot!(p_vals, lsq_data, label="Least squares")
plot!(p_vals, prescient_data, label="Prescient")
#-

## Plot the relative reconstruction error for Huber and prescient regression,
## zooming in on smaller values of p.
indices = findall(p_vals .<= 0.08);
plot(p_vals[indices], huber_data[indices], label="Huber")
plot!(p_vals[indices], prescient_data[indices], label="Prescient")


#=
							Even Fibonacci numbers

	Each  new term in the Fibonacci sequence is generated by adding the previous
	two terms. By starting with 1 and 2, the first 10 terms will be:

					1, 2, 3, 5, 8, 13, 21, 34, 55, 89, ...

	By  considering  the  terms  in  the  Fibonacci sequence whose values do not
	exceed four million, find the sum of the even-valued terms.

								Answer: 4613732
=#

function fibonacci(limit::Int)
    array = [1, 2]
    flag = true
    i = 2

    while flag
        new_value = array[i]+array[i-1]

        if new_value < limit
            push!(array, new_value)
            i += 1
        else
            flag = false
        end
    end

    return array
end

println(sum(filter(iseven, fibonacci(4_000_000))))


#=
(a, ya) (b, yb) (c, yc)
====
q(x) = p1 + p2*x + p3 * x^2
====
ya = p1 + p2*a + p3 * a^2
yb = p1 + p2*b + p3 * b^2
yc = p1 + p2*c + p3 * c^2
=====
lagrange’s interpolating polynomial:
q(x) = ya * ((x-b)(x-c))/((a-b)(a-c)) +
       yb * ((x-a)(x-c))/((b-a)(b-c)) +
       yc * ((x-a)(x-b))/((c-a)(c-b))
=====
q'(x) = 1/2 * (ya(b^2-c^2)+yb(c^2-a^2)+yc(a^2-b^2))/(ya(b-c) + yb(c-a) + yc(a-b))
=#

function quadratic_fit_search(f, a, b, c, n)
    ya, yb, yc = f(x), f(x), f(c);
    for i in 1:n-3
        x = 0.5 * (ya(b^2-c^2)+yb(c^2-a^2)+yc(a^2-b^2))/
                  (ya(b-c) + yb(c-a) + yc(a-b));
        yx = f(x);
        if x > b
            if yx > yb
                c, yc = x, yx;
            else
                a, ya, b, yb = b, yb, x, yx;
            end
        elseif x < b
            if yx > yb
                a, ya = x, yx;
            else
                c, yc, b, yb = b, yb, x, yx;
            end
        end
    end
    return (a, b, c);
end


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

"This routine runs a convergence test for the heat equation using either Backwards
Euler or Crank-Nicolson in time and finite differences in space. The exact solution
is generated using the method of manufactured solutions."

uexact(x,t) = log(2+sin(pi*x))*exp(-t)*t
dudx_exact(x,t) = ForwardDiff.derivative(x->uexact(x,t),x)
# du/dt - d^2u/dx^2 = f
f(x,t) = ForwardDiff.derivative(t->uexact(x,t),t) -
         ForwardDiff.derivative(x->dudx_exact(x,t),x)
u0(x) = uexact(x,0)
α(t) = uexact(-1,t)
β(t) = uexact(1,t)

# m = number of grid points
function build_FD(m,f,α,β)
    x = LinRange(-1,1,m+2)
    xint = x[2:end-1]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m),-1=>-ones(m-1),1=>-ones(m-1))

    function F(t)
        b = f.(xint,t)
        b[1] += α(t)/h^2
        b[m] += β(t)/h^2
        return b
    end
    return A,x,xint,h,F
end

function solve_backward_Euler(m, f, α, β, u0, T, dt)

    A,x,xint,h,F = build_FD(m,f,α,β)

    u = u0.(xint)
    Nsteps = ceil(Int,T/dt)
    dt = T/Nsteps
    for i = 1:Nsteps
        t = i*dt
        u .= (I + dt*A)\(u .+ dt*F(t))
    end
    return u,x,xint,h
end

function solve_Crank_Nicolson(m, f, α, β, u0, T, dt)

    A,x,xint,h,F = build_FD(m,f,α,β)

    u = u0.(xint)
    Nsteps = ceil(Int,T/dt)
    dt = T/Nsteps
    for i = 1:Nsteps
        tprev = (i-1)*dt
        t = i*dt
        u .= (I + .5*dt*A)\((I - .5*dt*A)*u + .5*dt*(F(t)+F(tprev)))
    end
    return u,x,xint,h
end

mvec = 2 .^ (2:8)
hvec = zeros(length(mvec))
hvec = Float64[]
err1 = Float64[]
err2 = Float64[]
for (i,m) in enumerate(mvec)

    T = 1.0
    dt = 5/m # proportional to h

    u,x,xint,h = solve_backward_Euler(m,f,α,β,u0,T,dt)
    push!(err1,maximum(@. abs(uexact(xint,T) - u)))

    u,x,xint,h = solve_Crank_Nicolson(m,f,α,β,u0,T,dt)
    push!(err2,maximum(@. abs(uexact(xint,T) - u)))

    push!(hvec,h)
end
plot(hvec,err1,marker=:dot,label="Max error (backwards Euler)")
plot!(hvec,err2,marker=:dot,label="Max error (Crank-Nicolson)")
plot!(hvec,hvec,linestyle=:dash,label="O(h)")
plot!(hvec,hvec.^2,linestyle=:dash,label="O(h^2)")
plot!(xaxis=:log,yaxis=:log,legend=:bottomright)


# Input data for generation of the cellular complex shown in figure of Data 2.1.2

V = [
    0.0 1.5 3.0 1.0 1.5 2.0 1.0 1.5 2.0 0.0 1.5 3.0;
    0.0 0.0 0.0 1.0 1.0 1.0 2.0 2.0 2.0 3.0 3.0 3.0
];
EV = [
    [1,  2], [2,  3], [ 4,  5], [ 5,  6],
    [7,  8], [8,  9], [10, 11], [11, 12],
    [1, 10], [4,  7], [ 6,  9], [ 3, 12],
    [2,  5], [8, 11]
];
FV = [
    [1, 2, 4, 5, 7, 8, 10, 11],
    [2, 3, 5, 6, 8, 9, 11, 12],
    [4, 5, 6, 7, 8, 9]
];

# The function K returns the Julia SparseArrays matrix providing
#  the characteristic matrix K𝑟 given as input an array CV
#  specifying each 𝑟-cell as array of vertex indices.

function K( CV )
    I = vcat( [ [k for h in CV[k]] for k=1:length(CV) ]...)
    J = vcat( CV...)
    X = Int8[1 for k=1:length(I)]
    return SparseArrays.sparse(I, J, X)
end

# Generation of the sparse binary matrices

copEV = K(EV);
copFV = K(FV);

Matrix(copEV)
Matrix(copFV)

if todisplay
    VV = [[k] for k in 1:size(V,2)];
    GL.VIEW( GL.numbering(1.0)( (V, [VV, EV]),GL.COLORS[1],0.1 ) );
end;

# Boundary matrix generation
# we construct the sparse matrix [𝜕2] from the product of characteristic matrices EV and the transposed FV

Matrix( copEV * copFV' )

copEF = (copEV * copFV') .÷ 2;

Matrix(copEF)

b1 = (copEF * sparse([1 1 1])') .% 2
b2 = (copEF * sparse([1 1 0])') .% 2

f1 = SparseArrays.findnz(b1)[1]
f2 = SparseArrays.findnz(b2)[1]

e1 = EV[f1]
e2 = EV[f2]

if todisplay
    GL.VIEW( GL.numbering(1.0)( (V, [VV, e1]),GL.COLORS[1],0.1 ) );
    GL.VIEW( GL.numbering(1.0)( (V, [VV, e2]),GL.COLORS[1],0.1 ) );
end;


#=
In this example we'll compare the optimization of a relatively simple problem: Logistic Regression.

Consider a binary classification dataset
  $\Omega = \{(x_i, y_i) \subset \mathbb{R}^p\times\{0,1\}:\ i = 1, \dots, n\}$.

The logistic regression problem defines a model
$h_{\beta}(x) = \sigma(\hat{x}^T \beta) = \sigma(\beta_0 + x^T\beta_{1:p})$,
where
$\hat{x} = \begin{bmatrix} 1 \\ x \end{bmatrix}$.
The value of $\beta$ is found by finding the maximum of the log-likelihood function

$$\ell(\beta) = \frac{1}{n} \sum_{i=1}^n y_i \ln \big(h_{\beta}(x_i)\big) + (1 - y_i) \ln\big(1 - h_{\beta}(x_i)\big).$$

We'll input the gradient of this function manually. It is given by

$$\nabla \ell(\beta) = \frac{1}{n} \sum_{i=1}^n \big(y_i - h_{\beta}(x_i)\big) \hat{x}_i = \frac{1}{n} \begin{bmatrix} e^T \\ X^T \end{bmatrix} (y - h_{\beta}(X)), $$

where $e$ is the vector with all components equal to 1.

Download [data.csv](https://github.com/abelsiqueira/julia-tutorial-at-nlesc/raw/main/assets/data.csv) to follow the example.

In Python, we can define the function $-\ell(\beta)$ to minimize, and its derivatives, as follows:

```python
import numpy as np
import pandas as pd
from scipy.optimize import minimize

df = pd.read_csv('data.csv')
X = df.iloc[:,0:-1].values
y = df.iloc[:,-1].values
n, p = X.shape

def sigmoid(t):
    return 1 / (1 + np.exp(-t))

def myfun(beta):
    hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
    return -np.sum(
        y * np.log(hbeta + 1e-8) + (1 - y) * np.log(1 - hbeta + 1e-8)
    ) / n

def myjac(beta):
    hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
    dif = hbeta - y
    return np.concatenate((np.array([np.sum(dif)]), X.T.dot(dif))) / n

beta0 = np.zeros(p + 1)
res = minimize(myfun, beta0, jac=myjac, method='l-bfgs-b', tol=1e-4)
beta = res.x
hbeta = sigmoid(beta[0] + X.dot(beta[1:]))
ypred = np.round(hbeta)
accuracy = sum(y == ypred) / n
print('accuracy = ', accuracy)
```
If you're using Jupyter, I also ask you to run
```
%%timeit
res = minimize(myfun, beta0, jac=myjac, method='l-bfgs-b', tol=1e-4)
```
So that we can compare the time

Here's a similar Julia version
=#
using CSV, DataFrames, Optim, LinearAlgebra

path = joinpath("assets", "data.csv")
df = DataFrame(CSV.File(path))
X = Matrix(df[:,1:end-1])
y = df[:,end]
n, p = size(X)

sigmoid(t) = 1 / (1 + exp(-t))

function myfun(β, X, y)
  @views hβ = sigmoid.(β[1] .+ X * β[2:end])
  out = sum(
    yᵢ * log(ŷᵢ + 1e-8) + (1 - yᵢ) * log(1 - ŷᵢ + 1e-8)
    for (yᵢ, ŷᵢ) in zip(y, hβ)
  )
  return -out / length(y)
end

function myjac(out, β, X, y)
  n = length(y)
  @views δ = (sigmoid.(β[1] .+ X * β[2:end]) - y) / n
  out[1] = sum(δ)
  out[2:end] .= X' * δ
  return out
end

output = optimize(
  β -> myfun(β, X, y),
  (out, β) -> myjac(out, β, X, y),
  zeros(p + 1),
  LBFGS(),
  Optim.Options(
    g_tol = 1e-4,
  ),
)

β = Optim.minimizer(output)
hβ = sigmoid.(β[1] .+ X * β[2:end])
ŷ = round.(hβ)
accuracy = sum(y .== ŷ) / n

#=
The beginning of the codes is similar: read the file, create `X` and `y`.

We then define the functions. Some main differences:
- Instead of writing the objective vectorized, I use a loop over the indices.
- To avoid creating a vector for `β[2:end]`, I use `@views` at the beginning of the line. The macro automatically transforms `β[2:end]` into `view(β, 2:end)`.
- The Julia `myjac` function receives an additional argument for the memory that it should reuse (`gx`) for the output.

Now, the specific part. In Python, we use SciPy's optimize, and in Julia we're using Optim.jl.
The arguments are similar, although Optim's arguments are positional.

We can benchmark Optim's code to compare against SciPy's optimize:
=#
using BenchmarkTools

β₀ = zeros(p + 1)
@benchmark optimize(
  β -> myfun(β, X, y),
  (out, β) -> myjac(out, β, X, y),
  β₀,
  LBFGS(),
  Optim.Options(
    g_tol = 1e-4,
  ),
)
#=
In Julia, we don't have a single optimization library.
In fact, I'm co-creator of the [JuliaSmoothOptimizers](https://juliasmoothoptimizers.github.io) organization, which "competes" against Optim.
We focus on tools for Nonlinear Optimization developers, instead of focusing on users, so our basic interface is a lot more raw.
On the other hand, we claim that our methods are faster, but notice my bias.

NLPModels alone doesn't have a simple user interface, so we create a struct `MyLogisticRegression` to hold the problem and to indicate the objective and gradient.
The struct fields are not obvious and you have to follow the docs of NLPModels to learn how to do it.
But you can check the speed gain.
=#

using JSOSolvers, Logging, NLPModels

struct MyLogisticRegression <: AbstractNLPModel{Float64, Vector{Float64}}
  meta :: NLPModelMeta{Float64, Vector{Float64}}
  counters :: Counters
  X
  y

  function MyLogisticRegression(X, y)
    meta = NLPModelMeta(p+1, x0=zeros(p+1))
    return new(meta, Counters(), X, y)
  end
end

NLPModels.obj(nlp :: MyLogisticRegression, x) = myfun(x, nlp.X, nlp.y)
NLPModels.grad!(nlp :: MyLogisticRegression, x, gx) = myjac(gx, x, nlp.X, nlp.y)

nlp = MyLogisticRegression(X, y)
output = lbfgs(nlp, atol=1e-4, rtol=1e-4)
println(output)

@benchmark with_logger(NullLogger()) do
  lbfgs(nlp, atol=1e-4, rtol=1e-4)
end

# TODO: Check progress of ManualNLPModels (check after 18-Oct)

#=
One noteworthy aspect of this comparison is that Python uses the L-BFGS-B Fortran algorithm, which is a reference implementation of a classic method.
The Julia versions are pure Julia.
=#


#= udemy.com code challenge
from Prof. Mike X Cohen's course in Linear Algebra
https://www.udemy.com/course/linear-algebra-theory-and-implementation

# Is the dot product commutative?
# A*B' == B*A' ?
=#

using LinearAlgebra

# generate two 100-element random row vectors, transposed from default column layout.
a = randn(100)'
b = randn(100)'

# display vectors (in truncated form)
println("Vector a with 100 elements:")
display(a)
println("\n")

println("Vector b of the same size:")
display(b)
println("\n")

# calculate dot products
dp1 = dot(a, b) # a * b' also works
dp2 = dot(b, a) # b * a'

# display them
println("Dot product of a and b is $(dp1)")
println("Dot product of b and a is $(dp2)\n")

# are they equal?
if dp1 == dp2
  println("The dot product of the 100-element vector is commutative.\n")
else
  println("Dot product is not commutative.\n")
end

# generate 1 row, 2 column array with random integers from -10:10
c = rand(-10:10, 1, 2)

d = rand(-10:10, 1, 2)

println("Array c = $c")
println("Array d = $d\n")

dp3 = c * d'
dp4 = d * c'

println("Dot product of c and d is $(dp3)")
println("Dot product of d and c is $(dp4)")

if dp3 == dp4
  println("Thus, the dot product of the 2-element vector is commutative.")
else
  println("Dot product is not commutative")
end


using Pkg
Pkg.activate(pwd())

# # Linear regression
# ## Loading and preparing data

using Plots
using StatsPlots
using RDatasets

iris = dataset("datasets", "iris")
iris[1:5,:]

# ### Exercise:
# We will simplify the goal and estimate the dependence of petal width on petal length. # Create the data $X$ (do not forget to add the bias) and the labels $y$.
#
# Make a graph of the dependence of petal width on petal length.
#
# ---
# ### Solution:

y = iris.PetalWidth
X = hcat(iris.PetalLength, ones(length(y)))

@df iris scatter(
    :PetalLength,
    :PetalWidth;
    label="",
    xlabel = "Petal length",
    ylabel = "Petal width"
)

# ---
#
# ## Training the classifier
# ### Exercise:
# Use the closed-form formula to get the coefficients $w$ for the linear regression. Then
# use the `optim` method derived in the previous lecture to solve the optimization problem
# via gradient descent. The results should be identical.
#
#---
#### Solution:

abstract type Step end

struct GD <: Step
    α::Real
end

optim_step(s::GD, f, g, x) = -s.α*g(x)

function optim(f, g, x, s::Step; max_iter=100)
    for i in 1:max_iter
        x += optim_step(s, f, g, x)
    end
    return x
end

#+

using LinearAlgebra

w = (X'*X) \ (X'*y)
g(w) = X'*(X*w-y)
w2 = optim([], g, zeros(size(X,2)), GD(1e-4); max_iter=10000)

norm(w - w2)

# ---
#
# ### Exercise:
# Write the dependence on the petal width on the petal length. Plot it in the previous
# graph.
#
# ---
# ### Solution:

f_pred(x::Real, w) = w[1]*x + w[2]
x_lims = extrema(iris.PetalLength) .+ [-0.1, 0.1]

@df iris scatter(
    :PetalLength,
    :PetalWidth;
    xlabel = "Petal length",
    ylabel = "Petal width",
    label = "",
    legend = :topleft,
)

plot!(x_lims, x -> f_pred(x,w); label = "Prediction", line = (:black,3))


function bisection(fun, a, b; threshold = 1e-10, maxiter = 100)
    root = (a+b)/2
    iter = 0
    while (abs(fun(root)) > threshold) && (iter < maxiter)
        if fun(a)*fun(root) < 0
            b = root
        else
            a = root
        end
        root = (a+b)/2
        iter += 1
    end
    if iter == maxiter
        println("Maximal number of iterations reached in bisection method.")
    end
    root
end

####################
# Newton iterations
#####################

"The abstract NonlinearSystem type represents a system of non-linear equations in n variables."
abstract type NonlinearSystem
end

# Default numeric type is Float64
# eltype(::Type{S}) where {S <: NonlinearSystem} = Float64

function residual(sys::NonlinearSystem, x)
    result = Array{eltype(sys)}(length(sys))
    residual!(result, sys, x)
end

function jacobian(sys::NonlinearSystem, x)
    J = Array{eltype(sys)}(size(sys))
    jacobian!(J, sys, x)
end

"""
Apply Newton iterations with the given starting values to find the roots of the given
system of equations. A convergence threshold and maximum number of iterations are optional.

The maximal step size in each iteration can be limited using `maxstep`.

Damped Newton iterations can be achieved by supplying damping parameter 0 < alpha < 1.
"""
function newton(sys::NonlinearSystem, x0; threshold = 1e-6, alpha = 1, maxiter = 100, maxstep = 1e5, verbose = false)
    n = length(x0)
    S = Array{eltype(sys)}(undef, n)
    J = Array{eltype(sys)}(undef, n, n)

    residual!(S, sys, x0)
    jacobian!(J, sys, x0)
    c = cond(J)
    threshold = max(threshold, eps(eltype(x0))*c)
    u = J \ S
    if norm(alpha*u) > maxstep
        u = u * maxstep/norm(u)
    end
    x1 = x0 - alpha*u

    iter = 1
    while norm(x1-x0) > threshold && iter < maxiter
        iter += 1
        x0 = x1
        residual!(S, sys, x0)
        jacobian!(J, sys, x0)
        u = J \ S
        if norm(alpha*u) > maxstep
            u = u * maxstep/norm(u)
        end
        x1 = x0 - alpha*u
    end
    if verbose
        if iter == maxiter
            println("Newton: maximum number of iterations, no convergence")
        else
            println("Newton: Converged to solution in ", iter, " iterations.")
        end
    end
    x1,iter,norm(x1-x0)
end

"Invoke Newton but restart with smaller maximal step size if an error is thrown."
function newton_with_restart(args...; maxstep = 1e5, maxiter = 100, verbose = false, options...)
    local nx,iter,normx
    nb_decreased = 0
    try
    nx,iter,normx = newton(args...; maxstep = maxstep, options...)
    catch e
        print("Error thrown by newton: ")
        println(e)
        iter = maxiter
    end
    while iter == maxiter
        verbose && println("Decreasing step size.")
        maxstep /= 2
        try
            nx,iter,normx = newton(args...; maxstep = maxstep, verbose=verbose, options...)
        catch e
            print("Error thrown by newton: ")
            println(e)
            iter = maxiter
        end
        nb_decreased += 1
        if nb_decreased > 20
            error("Too many step decreases in newton_with_restart. Aborting.")
        end
    end
    nx,iter,normx
end


"""
    lu_decompose(mat)
Decomposes a `n x n` non singular matrix into a lower triangular matrix (L) and an upper triangular matrix (U)
"""
function lu_decompose(mat)
	n = mat |> size |> first
	L = zeros(n, n)
	U = zeros(n, n)

	for i in 1:n
		for j in i:n
			s = 0
			for k in 1:i
				s += L[i,k] * U[k,j]
			end
			U[i,j] = mat[i,j] - s
		end

		for k in i:n
			if i == k
				L[i,i] = 1
    			else
				s = 0
				for j in 1:i
					s += L[k,j] * U[j,i]
				end
				L[k,i] = (mat[k,i] - s) / U[i,i]
			end
		end
	end

	return L, U
end


function errorCheck(a1,a2,p)
    if length(a1)!= length(a2)
        println("The arrays have different length")
        return -1
        end
        if p<1
            println("the value attributed to 'p' must be at least 1")
            return -1
        end
    return 0
end

function euclidianDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return sqrt((sum((a1-a2).^2)))
        end
end

function manhattanDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return sum(abs.((a1-a2)))
        end
end

function cosineDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return 1 - sum(a1.*a2)/(sqrt(sum(a1.^2))*sqrt(sum(a2.^2)))
        end
end

function minkowskiDistance(a1,a2,p)
        if errorCheck(a1,a2,p) !=-1
                return (sum(abs.(a1-a2).^p))^1/p
        end
end

function jaccardSimilarity(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return length(intersect(a1,a2))/length(union(a1,a2))
        end
end

function jaccardDistance(a1,a2)
        if errorCheck(a1,a2,1) !=-1
                return 1 - jaccardSimilarity(a1,a2)
        end
end


"""
    newton(f, fprime, x0, e, n)

Compute the root of the given function using Newton's method.
Uses the point at which tangent line intersects x-axis to find the next approximate of the root of function 'f',
as such it requires initial approximation `x0` and `fprime`, the derivative of function `f`.
Newton's method has very fast convergence provided a sufficiantly accurate initial approximation is chosen.

    newton(f::Function, fprime::Function, x0::Float64, e::Float64, n::Integer)

### OUTPUT:
  * `p` - approximation of the root of the function `f` (Float64) or,
  * `-1` - if `n` the maximum number of iterations was exceeded.
"""
function newton(f::Function, fprime::Function, x0::Float64, e::Float64, n::Integer)

    Afx = []

    for i in 1:n

        yprime = fprime(x0)

        if abs(yprime) < e
            warn("First derivative is zero")
            return x0
        end

        y = f(x0)
        x1 = x0 - y / yprime
        push!(Afx, f(x1))

        if abs(x1 - x0) < e
            return x1
        end

        x0 = x1
    end

    return -1
end


# Theil.jl

###### theil #####
"""
    theil(v)

    Compute the Theil Index of a vector `v`.

# Examples
```julia
julia> using Inequality
julia> theil([8, 5, 1, 3, 5, 6, 7, 6, 3])
0.10494562214323544
```
"""
theil(v::AbstractVector{<:Real})::Float64 = sum(v .* log.(v/Statistics.mean(v))) / sum(v)

###### weighted theil #####
"""
    theil(v, w)

Compute the Theil Index of a vector `v`, using weights given by a weight vector `w`.

Weights must not be negative, missing or NaN. The weights and data vectors must have the same length.

# Examples
```julia
julia> using Inequality
julia> theil([8, 5, 1, 3, 5, 6, 7, 6, 3], collect(0.1:0.1:0.9))
0.08120013911680612
```
"""
function theil(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64

    checks_weights(v, w)

    w = w[v .!= 0]
    v = v[v .!= 0]

    w = w/sum(w)
    v = v / sum(v .* w)
    return sum(v .* w .* log.(v))
end

function theil(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64

    checks_weights(v, w)

    w = w[v .!= 0]
    v = v[v .!= 0]

    w = w/w.sum
    v = v / sum(v .* w)
    return sum(v .* w .* log.(v))
end

"""
    wtheil(v, w)

Compute the Theil Index of `v` with weights `w`. See also [`theil`](@theil)
"""
wtheil(v::AbstractVector{<:Real}, w::AbstractVector{<:Real})::Float64 = theil(v,w)

wtheil(v::AbstractVector{<:Real}, w::AbstractWeights)::Float64 = theil(v,w)


"""
newton(f,dfdx,x₁)

Use Newton's method to find a root of `f` starting from `x₁`, where
`dfdx` is the derivative of `f`. Returns a vector of root estimates.
"""
function newton(f,dfdx,x₁)
    # Operating parameters.
    funtol = 100*eps();  xtol = 100*eps();  maxiter = 40;

    x = [x₁]
    y = f(x₁)
    dx = Inf   # for initial pass below
    k = 1

    while (abs(dx) > xtol) && (abs(y) > funtol) && (k < maxiter)
        dydx = dfdx(x[k])
        dx = -y/dydx            # Newton step
        push!(x,x[k]+dx)        # append new estimate

        k += 1
        y = f(x[k])
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
secant(f,x₁,x₂)

Use the secant method to find a root of `f` starting from `x₁` and
`x₂`. Returns a vector of root estimates.
"""
function secant(f,x₁,x₂)
    # Operating parameters.
    funtol = 100*eps();  xtol = 100*eps();  maxiter = 40;

    x = [x₁,x₂]
    y₁ = f(x₁); y₂ = 100;
    dx = Inf   # for initial pass below
    k = 2

    while (abs(dx) > xtol) && (abs(y₂) > funtol) && (k < maxiter)
        y₂ = f(x[k])
        dx = -y₂ * (x[k]-x[k-1]) / (y₂-y₁)   # secant step
        push!(x,x[k]+dx)        # append new estimate

        k += 1
        y₁ = y₂    # current f-value becomes the old one next time
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
newtonsys(f,jac,x₁)

Use Newton's method to find a root of a system of equations,
starting from `x₁`. The functions `f` and `jac should return the
residual vector and the Jacobian matrix, respectively. Returns
history of root estimates as a vector of vectors.
"""
function newtonsys(f,jac,x₁)
    # Operating parameters.
    funtol = 1000*eps();  xtol = 1000*eps();  maxiter = 40;

    x = [float(x₁)]
    y,J = f(x₁),jac(x₁)
    dx = Inf   # for initial pass below
    k = 1

    while (norm(dx) > xtol) && (norm(y) > funtol) && (k < maxiter)
        dx = -(J\y)             # Newton step
        push!(x,x[k] + dx)    # append to history
        k += 1
        y,J = f(x[k]),jac(x[k])
    end

    if k==maxiter
        @warn "Maximum number of iterations reached."
    end

    return x
end

"""
fdjac(f,x0,y0)

Compute a finite-difference approximation of the Jacobian matrix for
`f` at `x0`, where `y0`=`f(x0)` is given.
"""
function fdjac(f,x0,y0)

δ = sqrt(eps())   # FD step size
m,n = length(y0),length(x0)
if n==1
    J = (f(x0+δ) - y0) / δ
else
    J = zeros(m,n)
    Iₙ = I(n)
    for j = 1:n
        J[:,j] = (f(x0 + δ*Iₙ[:,j]) - y0) / δ
    end
end

return J
end

"""
levenberg(f,x₁,tol)

Use Levenberg's quasi-Newton iteration to find a root of the system
`f`, starting from `x₁`, with `tol` as the stopping tolerance in
both step size and residual norm. Returns root estimates as a
matrix, one estimate per column.
"""
function levenberg(f,x₁,tol=1e-12)

# Operating parameters.
ftol = tol;  xtol = tol;  maxiter = 40;

x = zeros(length(x₁),maxiter)
x = [float(x₁)]
fₖ = f(x₁)
k = 1;  s = Inf;
Aₖ = fdjac(f,x₁,fₖ)   # start with FD Jacobian
jac_is_new = true

λ = 10;
while (norm(s) > xtol) && (norm(fₖ) > ftol) && (k < maxiter)
    # Compute the proposed step.
    B = Aₖ'*Aₖ + λ*I
    z = Aₖ'*fₖ
    s = -(B\z)

    xnew = x[k] + s
    fnew = f(xnew)

    # Do we accept the result?
    if norm(fnew) < norm(fₖ)    # accept
        y = fnew - fₖ
        push!(x,xnew)
        fₖ = fnew
        k += 1

        λ = λ/10   # get closer to Newton
        # Broyden update of the Jacobian.
        Aₖ = Aₖ + (y-Aₖ*s)*(s'/(s'*s))
        jac_is_new = false
    else                       # don't accept
        # Get closer to steepest descent.
        λ = 4λ
        # Re-initialize the Jacobian if it's out of date.
        if !jac_is_new
            Aₖ = fdjac(f,x[k],fₖ)
            jac_is_new = true
        end
    end
end

if (norm(fₖ) > 1e-3)
    @warn "Iteration did not find a root."
end

return x
end


#
###
# Lagrange polynomial matrices
###

"""
    Compute the Lagrange interpolation matrix from xi to xo.
    lagrange_poly_interp_mat(xₒ,xᵢ)
"""
function lagrange_interp_mat(xo,xi)

    no = length(xo)
    ni = length(xi)

    a = ones(1,ni)
    for i=1:ni
        for j=1:(i-1)  a[i]=a[i]*(xi[i]-xi[j]); end
        for j=(i+1):ni a[i]=a[i]*(xi[i]-xi[j]); end
    end
    a = 1 ./ a # Barycentric weights

    J = zeros(no,ni)
    s = ones(1,ni)
    t = ones(1,ni)
    for i=1:no
        x = xo[i]
        for j=2:ni
            s[j]      = s[j-1]    * (x-xi[j-1]   )
            t[ni+1-j] = t[ni+2-j] * (x-xi[ni+2-j])
        end
        J[i,:] = a .* s .* t
    end

    return J
end

"""
 Compute derivative matrix for lagrange
 interpolants on points [x]
"""
function lagrange_deriv_mat(x)

    n = length(x)

    a = ones(1,n)
    for i=1:n
        for j=1:(i-1) a[i]=a[i]*(x[i]-x[j]) end
        for j=(i+1):n a[i]=a[i]*(x[i]-x[j]) end
    end
    a = 1 ./ a # Barycentric weights

    # diagonal elements
    D = x .- x'
    for i=1:n D[i,i] = 1. end
    D = 1 ./ D
    for i=1:n
        D[i,i] = 0.
        D[i,i] = sum(D[i,:])
    end

    # off-diagonal elements
    for j=1:n for i=1:n
        if(i!=j) D[i,j] = a[j] / (a[i]*(x[i]-x[j])) end
    end end

    return D
end


################################################################################

"""
    euclDist(arr::Array{T, 1}, h::Array{T, 1}) where {T <: Number}

# Description
Euclidean distance.

# Examples
```
julia> euclDist(collect(1:10), fill(5, 10))
9.219544457292887
```

See also: [`bhattDist`](@ref), [`amplitude`](@ref)
"""
function euclDist(arr::Array{T,1}, h::Array{T,1}) where {T <: Number}
  return (arr .- h) .^ 2 |> sum |> sqrt
end

"""
    bhattDist(arr::Array{T, 1}, h::Array{T, 1}) where {T <: Number}

# Description
Bhattacharyya distance.

# Examples
```
julia> bhattDist(collect(1:10), fill(5, 10))
-3.936532135073928
```

See also: [`euclDist`](@ref), [`amplitude`](@ref)
"""
function bhattDist(arr::Array{T,1}, h::Array{T,1}) where {T <: Number}
  return -log((arr .* h .|> sqrt |> sum) + 1)
end

"""
    amplitude(arr::Array{T, 1}) where {T <: Number}

# Description
Amplitude.

# Examples
```
julia> amplitude(collect(1:10))
19.621416870348583
```

See also: [`euclDist`](@ref), [`bhattDist`](@ref)
"""
function amplitude(arr::Array{T,1}) where {T <: Number}
  return arr .^ 2 |> sum |> sqrt
end

################################################################################


# Continuation of ex 7.13, Least Squares WRM

using NumericalMethodsforEngineers
using Test

@sym begin
  ClearAll(xi, yi, N1, Y, Ydotdot, C11, ytilde1, ytilde2)
  xi = [0, 1//3, 2//3, 1]
  yi = [0, a, b, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  #
  # Can be re-formulated as Y(x) = F(x) + C1(a) * Ψ1(x) + C2(a) * Ψ2(x)
  #
  # F(x_) := (1/2)*(9x^3 - 9x^2 + 2x)
  # C1 = (27/2)*(a-b)
  Ψ1(x_) := x^3 - x^2
  # C2 = -(9/2)*(2a-b)
  Ψ2(x_) := x^2 - x
  #
  # Y(x_) := F(x) + C1(a)*Ψ1(x) + C2(a)*Ψ2(x)
  #
  Ydotdot(x_) = D(Y(x), x, 2)
  R(x_) := Ydotdot(x) - 3*x - 4*Y(x)
  Rdota(x_) = D(R(x), a)
  Rdotb(x_) = D(R(x), b)
  sol = Solve([Integrate(R(x)*Rdota(x), [x, 0, 1]), Integrate(R(x)*Rdotb(x), [x, 0, 1])], [a, b])
  SetJ(r, ToString(Simplify(R(x))))
  SetJ(a, ToString(sol[1][1][2]))
  SetJ(b, ToString(sol[1][2][2]))
  ytilde1 = Simplify(Y(x) ./ (a => sol[1][1][2]))
  ytilde2 = Simplify(ytilde1 ./ (b => sol[1][2][2]))
  SetJ(y, ToString(Simplify(Expand(ytilde2))));
end

println("\n\nExample 7.17b: y'' = 3x + 4y, y(0)=0, y(1)=1")
println("by 2-point least squares Weighted Residual Method")
@sym Println("\nY(x) = $(Simplify(Expand(Y(x))))\n")
@sym Println("R(x) = $(R(x))\n")
@eval a = $(Meta.parse(a))
@eval b = $(Meta.parse(b))
println("(a,b) = $((a, b))\n")
println("C1 = $((27/2*(a-b)))\n")
println("C2 = $(-(9/2*(2a-b)))\n")
@eval rf_2pt_leastsquares(x, a, b) = $(Meta.parse(r))
@eval ytilde_2pt_leastsquares(x) = $(Meta.parse(y))
println()

rf_2pt_leastsquares_1(x) = rf_2pt_leastsquares(x, a, b)
println()

@test r == "-9 - 45a + 36b + 20x - 63b*x + 45a*x + 18*x^2 - 72b*x^2 + 90a*x^2 - 18x^3 - 54a*x^3 + 54b*x^3"
@test y == "(1/9693)*x*(2420 - 3311x + 10584*x^2)"
#@test (quadgk(rf_2pt_leastsquares_1, 0, 1))[1] < 5*eps()


#!/usr/bin/env julia

"""
# Problem 1: Multiples of 3 and 5

If we list all the natural numbers below 10 that are multiples of 3 or 5, we
get 3, 5, 6 and 9. The sum of these multiples is 23.

Find the sum of all the multiples of `m` or `n` below `limit`.
"""
module Problem001

include("Sequences.jl")

"""
    solve(; m::Integer = 3, n::Integer = 5, limit::Integer = 1000)

Returns the solution for [`Problem001`](@ref) with the given parameters.

## Preconditions
- `m > 0`
- `n > 0`
- `limit > 0`
"""
function solve(; m::Integer = 3, n::Integer = 5, limit::Integer = 1000)
    m_sum = sum_divisible_by(m, limit)
    n_sum = sum_divisible_by(n, limit)
    lcm_sum = sum_divisible_by(lcm(m, n), limit)
    return m_sum + n_sum - lcm_sum
end

"""
    sum_divisible_by(n::Integer, limit::Integer)

Returns the sum of natural numbers below `limit` that are divisible by `n`.

## Preconditions
- `n > 0`
- `limit > 0`

## Examples
```jldoctest
julia> sum_divisible_by(3, 10)
18

julia> sum_divisible_by(5, 10)
5

julia> sum_divisible_by(5, 5)
0
```
"""
function sum_divisible_by(n::Integer, limit::Integer)
    return Sequences.arithmetic_series(n, (limit - 1) ÷ n, n)
end

end


#Function that group the mainly statistics moments that include nan values

function nanmean(vector::Vector)
N=length(vector);
s=0.0;                                        #--sample mean sum
m=0.0;                                        #--counter for the sample size
  for i=1:N
     if isnan(vector[i])==0;                  #--make sure to sum numeric values
        m=m+1;
        s=s+vector[i];
     end
  end
mu=s/m;
mu
end

#In nanvar.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nanvar(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         m=m+1;
         delta=vector[i]-mu;
         mu=mu+delta/m;
         M2=M2+delta*(vector[i]-mu);
      end
 end
 if m<2;
    return NaN
 else
    return M2/m
 end
end

#In nanskew.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nanskew(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2
M3=0.0;            #---Intitialize sum(x_i-mu)^2
M4=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         n1=m;
         m=m+1;
         delta=vector[i]-mu;
         delta_m=delta/m;
         delta_m2=delta_m*delta_m;
         term1=delta*delta_m*n1;
         mu=mu+delta_m;
         M3=M3+term1*delta_m*(m-2)-3*delta_m*M2
         M2=M2+term1;
      end
 end
skewness=sqrt(m)*M3/M2^(3/2);
return skewness
end

#In nankurt.jl
#https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Higher-order_statistics
function nankurt(vector::Array)
N=length(vector);
m=0.0;             #---Initialize counter for samples
mu=0.0;            #---Initialize mean
M2=0.0;            #---Intitialize sum(x_i-mu)^2
M3=0.0;            #---Intitialize sum(x_i-mu)^2
M4=0.0;            #---Intitialize sum(x_i-mu)^2

 for i=1:N
      if isnan(vector[i])==0;                  #--make sure to sum numeric values
         n1=m;
         m=m+1;
         delta=vector[i]-mu;
         delta_m=delta/m;
         delta_m2=delta_m*delta_m;
         term1=delta*delta_m*n1;
         mu=mu+delta_m;
         M4=M4+term1*delta_m2*(m*m-3*m+3)+6*delta_m2*M2-4*delta_m*M3
         M3=M3+term1*delta_m*(m-2)-3*delta_m*M2
         M2=M2+term1;
      end
 end
#kurtosis=(m*M4)/(M2*M2)-3;      #--normalized
kurtosis=(m*M4)/(M2*M2);
return kurtosis
end


# This file investigates MSE with respect to free variables

using FractalTools
using GeometryBasics
using Makie

# Construct interpolation data
f = FractalTools.ackley
α = 10
ngon = [BigFloat.([-1., -1.]), BigFloat.([1., -1]), BigFloat.([0., 1.])] * α
npts = 100
pts = getdata(f, ngon, npts)

# Construct test data
tpts = getdata(ngon, npts)
ntpts = length(tpts)

# Compute errors
fvals = map(pt -> f(pt...), tpts)
freevars = 0.001 : 0.001 : 0.05
mse = map(freevars) do freevar
    interp = interpolate(pts, Interp2D(freevar))
    ivals = map(pt -> interp(pt...), tpts)
    sum((fvals - ivals).^2) / ntpts
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="MSE", title="2D Interpolation MSE")
stem!(ax, freevars, mse, color=:black)
save(joinpath(@__DIR__, "interp2d_mse.png"), fig)
display(fig)

# using FractalTools
# using GeometryBasics
# using Makie

# # Construct interpolation data
# f(x, y) = -20*exp(-0.2 * √(0.5*(x^2+y^2))) -
#             exp(0.5*(cos(2π*x)+cos(2π*y))) +
#             MathConstants.e + 20

# # f(x, y) = x^2 + y^2 + 1

# # f(x, y) = sin(x) * cos(y)
# ngon = Triangle(
#     Point(BigFloat(-5.), BigFloat(-5.)),
#     Point(BigFloat(5), BigFloat(-5.)),
#     Point(BigFloat(0), BigFloat(5.)))

# # ngon = [BigFloat.([-5,5]),
# #         BigFloat.([-5,-5]),
# #         BigFloat.([5,-5]),
# #         BigFloat.([5,5])
# # ]
# npts = 100
# pts = getdata(f, ngon, npts)

# # Construct test data
# tpts = getdata(ngon, npts)
# ntpts = length(tpts)

# # Compute errors
# fvals = map(pt -> f(pt...), tpts)
# freevars = 0.001 : 0.001 : 0.05
# mse = map(freevars) do freevar
#     interp = interpolate(pts, Interp2D(freevar))
#     ivals = map(pt -> interp(pt...), tpts)
#     sum((fvals - ivals).^2) / ntpts
# end

# # Plot mse
# fig = Figure()
# ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="MSE", title="2D Interpolation MSE")
# stem!(ax, freevars, mse, color=:black)
# save(joinpath(@__DIR__, "interp2d_mse.png"), fig)
# display(fig)


include("misc.jl")

function findMin(funObj,w;maxIter=100,epsilon=1e-2,derivativeCheck=false,verbose=true)
	# funObj: function that returns (objective,gradient)
	# w: initial guess
	# maxIter: maximum number of iterations
	# epsilon: stop if the gradient gets below this
	# derivativeCheck: whether to check against numerical gradient

	# Evalluate the intial objective and gradient
	(f,g) = funObj(w)

	# Optionally check if gradient matches finite-differencing
	if derivativeCheck
		g2 = numGrad(funObj,w)

		if maximum(abs.(g-g2)) > 1e-4
			@show([g g2])
			@printf("User and numerical derivatives differ\n")
			sleep(1)
		else
			@printf("User and numerical derivatives agree\n")
		end
	end

	# Initial step size and sufficient decrease parameter
	gamma = 1e-4
	alpha = 1
	for i in 1:maxIter

		# Try out the current step-size
		wNew = w - alpha*g
		(fNew,gNew) = funObj(wNew)

		# Decrease the step-size if we increased the function
		gg = dot(g,g)
		while fNew > f - gamma*alpha*gg

			if verbose
				@printf("Backtracking\n")
			end

			# Fit a degree-2 polynomial to set step-size
			alpha = alpha^2*gg/(2(fNew - f + alpha*gg))

			# Try out the smaller step-size
			wNew = w - alpha*g
			(fNew,gNew) = funObj(wNew)
		end

		# Guess the step-size for the next iteration
		y = gNew - g
		alpha *= -dot(y,g)/dot(y,y)

		# Sanity check on the step-size
		if (!isfinitereal(alpha)) | (alpha < 1e-10) | (alpha > 1e10)
			alpha = 1
		end

		# Accept the new parameters/function/gradient
		w = wNew
		f = fNew
		g = gNew

		# Print out some diagnostics
		gradNorm = norm(g,Inf)
		if verbose
			@printf("%6d %15.5e %15.5e %15.5e\n",i,alpha,f,gradNorm)
		end

		# We want to stop if the gradient is really small
		if gradNorm < epsilon
			if verbose
				@printf("Problem solved up to optimality tolerance\n")
			end
			return w
		end
	end
	if verbose
		@printf("Reached maximum number of iterations\n")
	end
	return w
end

function findMinL1(funObj,w,lambda;maxIter=100,epsilon=1e-2)
	# funObj: function that returns (objective,gradient)
	# w: initial guess
	# lambda: value of L1-regularization parmaeter
	# maxIter: maximum number of iterations
	# epsilon: stop if the gradient gets below this

	# Evalluate the intial objective and gradient
	(f,g) = funObj(w)

	# Initial step size and sufficient decrease parameter
	gamma = 1e-4
	alpha = 1
	for i in 1:maxIter

		# Gradient step on smoooth part
		wNew = w - alpha*g
		# Proximal step on non-smooth part
		wNew = sign.(wNew).*max.(abs.(wNew) - lambda*alpha,0)
		(fNew,gNew) = funObj(wNew)

		# Decrease the step-size if we increased the function
		gtd = dot(g,wNew-w)
		while fNew + lambda*norm(wNew,1) > f + lambda*norm(w,1) - gamma*alpha*gtd
			@printf("Backtracking\n")
			alpha /= 2

			# Try out the smaller step-size
			wNew = w - alpha*g
			wNew = sign.(wNew).*max.(abs.(wNew) - lambda*alpha,0)
			(fNew,gNew) = funObj(wNew)
		end

		# Guess the step-size for the next iteration
		y = gNew - g
		alpha *= -dot(y,g)/dot(y,y)

		# Sanity check on the step-size
		if (!isfinitereal(alpha)) | (alpha < 1e-10) | (alpha > 1e10)
			alpha = 1
		end

		# Accept the new parameters/function/gradient
		w = wNew
		f = fNew
		g = gNew

		# Print out some diagnostics
		optCond = norm(w-sign.(w-g).*max.(abs.(w-g)-lambda,0),Inf)
		@printf("%6d %15.5e %15.5e %15.5e\n",i,alpha,f+lambda*norm(w,1),optCond)

		# We want to stop if the gradient is really small
		if optCond < epsilon
			@printf("Problem solved up to optimality tolerance\n")
			return w
		end
	end
	@printf("Reached maximum number of iterations\n")
	return w
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Foundations of machine learning")

## Run this cell to load the graphics packages
using Plots; gr()

# ## Adding a function parameter

#-

# In the last notebook, we saw an example of adding **parameters** to functions. These are values that control the behavior of a function. Let's look at that in some more detail.

#-

# Let's go back to our original version of the σ function:

σ(x) = 1 / (1 + exp(-x))

# Instead of working with a single function, we can work with a whole class (set) of functions that look similar but differ in the value of a **parameter**. Let's make a new function that uses the previous $\sigma$ function, but also has a parameter, $w$. Mathematically, we could write
#
# $$f_w(x) = f(x; w) = \sigma(w \, x).$$
#
# (Here, $w$ and $x$ are multiplied in the argument of $\sigma$; we could write $w \times x$ or $w * x$, or even $w \cdot x$, but usually the symbols are not written.)
#
# Mathematically speaking, we can think of $f_w$ as a different function for each different value of the parameter $w$.

#-

# In Julia, we write this as follows:

f(x, w) = σ(w * x)

# Note that Julia just treats parameters as additional *arguments* to the function; the function `f` has two arguments, the value of `x` and the value of `w` that we want to use.

#-

# We can now investigate the effect of $w$ interactively. To do so, we need a way of writing in Julia "the function of one variable $x$ that we obtain when we fix the value of $w$". We write this as an "anonymous function", as we saw in the notebook on functions:
#
#     x -> f(x, w)
#
# We can read this as "the function that maps $x$ to the value of $f(x, w)$, for a value of $w$ that was previously given".

#-

# Now we are ready to draw the function. For each plot, we *fix* a value of the parameter $w$ and draw the resulting function as a function of $x$. We can interactively modify the value of $w$, and plot the new function that comes out:

## Try manipulating `w` between -2 and 2
w = 0.0
plot(x->f(x, w), -5, 5, ylims=(0,1), label="sigmoid")
plot!(x->(x>0), -5,5, label="Square Wave")

# #### Exercise 1
#
# Try writing your own function that takes a parameter. Start by copying and executing
#
# ```julia
# square(x) = x^2
# ```
#
# Then use `square` to declare a new function `square_and_scale` that takes two inputs, `a` and `x` such that
#
# $$\mathrm{square\_and\_scale}(x; a) := a \cdot x^2$$

#-

# #### Solution
#
# You should have declared the following function:
#
# ```julia
# square_and_scale(x, a) = a * x^2
# ```
#
# We can write some tests as follows, using the approximate equality operator, `≈`, written as `\approx<TAB>`:
#
# ```julia
# square_and_scale(1, 1) ≈ 1
# square_and_scale(1, 2) ≈ 2
# square_and_scale(2, 1) ≈ 4
# square_and_scale(2, 2) ≈ 8
# ```

#-

# #### Exercise 2

#-

# Once you have declared `square_and_scale`, uncomment the code below and see how the parameter `a` scales the function `square` :

## x = -10:10
## a = 5.0 # Try manipulating a here
## plot(x, square.(x), label="x^2")
## plot!(x, square_and_scale.(x, a), ls=:dash, label="ax^2")

# ## Fitting a function to data

#-

# As we saw in the previous notebook, what we would like to do is use the fact that we now have a parameter in our function in order to do something useful! Namely, we want to model data with it.

#-

# So suppose that we are given a single data point $(x_0, y_0) = (2, 0.8)$. We can try to "fit" the function $f_w$ by adjusting the parameter $w$ until the function passes through the data.
#
# **Game**: Change `w` until the graph of the function hits the data point. Which value of $w$ does that correspond to?

x0, y0 = 2, 0.8

w = 0.0 # try mainipulatoing `w` between -2 and 2
plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f")
scatter!([x0], [y0], label="data")

# ## Quantifying how far we are from the goal: the *loss function*

#-

# We can see visually when the graph of the function passes through the data point. But the goal is to be able to automate this process so that the computer can do it unaided.
#
# So we will need a more precise way of deciding and quantifying (i.e. measuring with a number) *how far away we are from the goal*; here, the goal means hitting the data point with the function.

#-

# #### Exercise 3
#
# Can you think of a way of measuring how far away our function is from the data point?

#-

# ### Defining the loss function

#-

# We need to measure how far away the curve is from the data point when we choose a particular value of $w$.
# One way to do this is by finding the vertical distance $d$ from the curve to the data point.
#
# Instead of just taking the distance, it is common to take the *square* of the distance, $d^2$.
#
# Since we are taking the vertical distance, we need the distance at the given value of $x_0$ where the data point lies. For a given value of the parameter $w$, the height of the point on the curve with that value of $x_0$ is $f(x_0, w)$.
#
# So we take
# $$d := y_0 - f(x_0, w)$$
#
# and
# $$d^2 = [y_0 - f(x_0, w)]^2.$$
#
# This is our measure of distance. It will change when $w$ changes -- in other words, it is itself a *function of $w$*; we will denote this function by $L(w)$, and call it the **loss function**:
#
# $$L(w) := [y_0 - f(x_0, w)]^2.$$
#
# So the goal is to find the value $w^*$ of $w$ where the loss function is *least*; in other words, we need to *minimize* the loss function!
#
# (Another name for a loss function is a *cost function*.)

#-

# #### Exercise 4
#
# (a) Define the loss function `L(w)` in Julia.
#
# (b) Draw the data point and the function `x -> f(x, w)`. Also draw a vertical line from the data point to the function `x -> f(x, w)`.
#
# (c) Make the plot interactive.
#
# (d) Add as the plot title the value of the loss function for the current value of $w$.
#
# (e) Use the slider to find the value $w^*$ of $w$ for which the loss function reaches its minimum value. What is $w^*$? What is the value of the loss function there, $L(w^*)$?

#-

# #### Solution

#-

# (a) We have data

x0, y0 = 2, 0.8

# We define the loss function as follows:

L(w) = (y0 - f(x0, w))^2

# (b-d): The visualization can be written as follows

w = 0.0 # try manipulating w here between -2 and 2

plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f_w", lw=3)  # function

scatter!([x0], [y0], label="data")
plot!([x0, x0], [y0, f(x0, w)], label="loss")  # vertical line

title!("L(w) = $(round(L(w); sigdigits = 5))")

# ## What does the loss function look like?

#-

# The loss function $L(w)$ tells us how far away the function $f_w$ is from the data when the parameter value is $w$, represented visually as the vertical line in the previous plot. When the data are fixed, this is a function only of the parameter $w$. What does this function look like as a function of $w$? Let's draw it!

#-

# #### Exercise 5
#
# Draw the function $L(w)$ as a function of $w$.

#-

# #### Solution

plot(L, -3, 3, xlabel="w", ylabel="L(w)", ylims=(0, 0.7), label="L")

# ### Features of the loss function

#-

# This graph quantifies how far we are from the data point for a given value of $w$.
# What features can we see from the graph?
#
# Firstly, we see that $L(w)$ is always bigger than $0$, for any value of $w$. This is because we want $L$ to be some kind of measure of *distance*, and distances cannot be negative.
#
# Secondly, we see that there is a special value $w^*$ of $w$ where the function $L$ reaches its minimum value. In this particular case, it actually reaches all the way down to $0$!
# This means that the original function $f$ (the one we manipulated above) passes exactly through the data point $(x_0, y_0)$.

#-

# #### Exercise 6
#
# Draw a zoomed-in version of the graph to find the place $w^*$ where the function hits $0$ more precisely.

#-

# ### A different way of defining the loss function

#-

# **Why did we use such a complicated function $L$ with those squares inside?** We could instead just have used the absolute distance, instead of the distance squared, using the *absolute value* function, written mathematically as $| \cdot |$, and in Julia as `abs`.

#-

# #### Exercise 7
#
# Define a new loss function, `L_abs`, using the absolute value, and see what it looks like.

#-

# #### Solution

L_abs(w) = abs(y0 - f(x0, w))

plot(L_abs, -3, 3, xlabel="w", ylabel="L_abs(w)", ylims=(0, 0.7), label="L_abs")

# Now we see why it was previously generally preferred to use squares: using the absolute value gives a cost function that is *not smooth*. This makes it difficult to use methods from calculus to find the minimum. Nonetheless, using non-smooth functions is very common in machine learning nowadays.

Courses/Foundations of machine learning/0500.Building-models-quiz.jl


# --- Imports

import Distributions
using Distributions: MvNormal, Uniform, ContinuousUnivariateDistribution, cdf, pdf
using HypothesisTests
using LinearAlgebra
using Plots
using Random

# --- Generate sample of vectors drawn from a uniform distribution on a unit circle

n = 2
num_samples_2d = 50000

dist = Uniform(-π, π)
num_vectors = num_samples_2d
theta = rand(dist, num_vectors)
vectors = transpose(hcat(cos.(theta), sin.(theta)));

# --- Plot histogram of angles

thetas = map(i -> atan(vectors[:, i][2], vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 25
hist_bins = range(-π, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_2d = 0.5 / π
println("pdf(θ): $(pdf_2d)")

# Display plot
display(plt)

# --- Perform test for uniformity on the circle

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution2D <: ContinuousUnivariateDistribution end
Distributions.pdf(dist::ThetaDistribution2D, x::Real) = 0.5 / π
Distributions.cdf(dist::ThetaDistribution2D, x::Real) = 0.5 * (x + π) / π

theta_dist = ThetaDistribution2D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit circle

n = 2
num_samples_2d = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples_2d
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 25
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_2d = 1 / π
println("pdf(θ): $(pdf_2d)")

# Display plot
display(plt)

# --- Perform test for uniformity on the circle

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution2D <: ContinuousUnivariateDistribution end
Distributions.pdf(dist::ThetaDistribution2D, x::Real) = 1 / π
Distributions.cdf(dist::ThetaDistribution2D, x::Real) = x / π

theta_dist = ThetaDistribution2D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 3
num_samples_3d = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples_3d
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_3d(x) = 0.5 * sin(x)
plt = plot!(pdf_3d, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the sphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistribution3D <: ContinuousUnivariateDistribution end
Distributions.cdf(dist::ThetaDistribution3D, x::Real) = 0.5 * (1 - cos(x))

theta_dist = ThetaDistribution3D()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 20
num_samples = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_nd(x) = (2^(n-2) / π / binomial(n-2, (n-2) ÷ 2)) * sin(x)^(n-2)
plt = plot!(pdf_nd, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the hypersphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistributionND <: ContinuousUnivariateDistribution end

function Distributions.cdf(dist::ThetaDistributionND, x::Real)
    k = n-2
    sgn = (-1)^(k÷2)
    value = 0
    for j = 0:(k÷2 - 1)
        value += (-1)^j * binomial(k, j) * sin((k - 2*j) * x) / (k - 2*j)
    end
    value *= 2
    value += sgn * binomial(k, k÷2) * x  # contribution from i = k÷2 term
    value *= sgn / π / binomial(k, k÷2)

    return value
end

theta_dist = ThetaDistributionND()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)

# --- Generate sample of vectors drawn from a uniform distribution on a unit sphere

n = 45
num_samples = 50000

# Generate vectors
dist = MvNormal(zeros(n), ones(n))
num_vectors = num_samples
vectors = rand(dist, num_vectors)
for i = 1:num_vectors
    vectors[:, i] /= norm(vectors[:, i])
end

# --- Plot histogram of angles

thetas = map(i -> atan(norm(vectors[:, i][2:end]), vectors[:, i][1]), 1:num_vectors)

num_hist_bins = 50
hist_bins = range(0, π; length=num_hist_bins)
hist = histogram(thetas; bins=hist_bins, normalize=true)
plt = plot(hist)

pdf_nd(x) = (2^(n-2) / π / binomial(n-2, (n-2) ÷ 2)) * sin(x)^(n-2)
plt = plot!(pdf_nd, 0, π)

# Display plot
display(plt)

# --- Perform test for uniformity on the hypersphere

# Analytical formula for distribution over angle from x-axis
struct ThetaDistributionND <: ContinuousUnivariateDistribution end

function Distributions.cdf(dist::ThetaDistributionND, x::Real)
    k = n-2
    value = 0
    norm_ = 0
    for j = 0:(k-1)÷2
        coef = (-1)^j * binomial(k, j) / (k - 2*j)
        value += coef * (1 - cos((k - 2*j) * x))
        norm_ += coef
    end
    value /= 2 * norm_

    return value
end

theta_dist = ThetaDistributionND()

# Perform Anderson-Darling Test
test_results = OneSampleADTest(thetas, theta_dist)
println(test_results)

test_results = ExactOneSampleKSTest(thetas, theta_dist)
println(test_results)

test_results = ApproximateOneSampleKSTest(thetas, theta_dist)
println(test_results)


using Statistics, StatsBase, LinearAlgebra, DataFrames, CSV
df = CSV.read("../data/3featureData.csv",header=false)
n, p = size(df)
println("Number of features: ", p)
println("Number of observations: ", n)
X = convert(Array{Float64,2},df)
println("Dimensions of data matrix: ", size(X))

xbarA = (1/n)*X'*ones(n)
xbarB = [mean(X[:,i]) for i in 1:p]
xbarC = sum(X,dims=1)/n
println("\nAlternative calculations of (sample) mean vector: ")
@show(xbarA), @show(xbarB), @show(xbarC)

Y = (I-ones(n,n)/n)*X
println("Y is the de-meaned data: ", mean(Y,dims=1))

covA = (X .- xbarA')'*(X .- xbarA')/(n-1)
covB = Y'*Y/(n-1)
covC = [cov(X[:,i], X[:,j]) for i in 1:p, j in 1:p]
covD = [cor(X[:,i], X[:,j])*std(X[:,i])*std(X[:,j]) for i in 1:p, j in 1:p]
covE = cov(X)
println("\nAlternative calculations of (sample) covariance matrix: ")
@show(covA), @show(covB), @show(covC), @show(covD), @show(covE)

ZmatA = [(X[i,j] - mean(X[:,j]))/std(X[:,j]) for i in 1:n, j in 1:p ]
ZmatB = hcat([zscore(X[:,j]) for j in 1:p]...)
println("\nAlternate computation of Z-scores yields same matrix: ",
	maximum(norm(ZmatA-ZmatB)))
Z = ZmatA

corA = covA ./ [std(X[:,i])*std(X[:,j]) for i in 1:p, j in 1:p]
corB = covA ./ (std(X,dims = 1)'*std(X,dims = 1))
corC = [cor(X[:,i],X[:,j]) for i in 1:p, j in 1:p]
corD = Z'*Z/(n-1)
corE = cov(Z)
corF = cor(X)
println("\nAlternative calculations of (sample) correlation matrix: ")
@show(corA), @show(corB), @show(corC), @show(corD), @show(corE), @show(corF);


function parabolicinterp(f::Function, xl::Number, xm::Number, xu::Number, ea::Number, maxIter::Int64)
  # 1D optimisation using parabolic interpolation

  xo = NaN; # Current estimate of the optimum
  fxm = f(xm);
  fxu = f(xu);
  fxl = f(xl);

  for i = 1:maxIter
    xoold = xo; # xoxo

    # Solution for xo using simultaneous equations
    fracNumerator = ((xm - xl)^2 * (fxm-fxu)) - ((xm - xu)^2 * (fxm - fxl));
    fracDenominator = ((xm - xl) * (fxm - fxu)) - ((xm - xu) * (fxm - fxl));
    xo = xm - 0.5 * (fracNumerator / fracDenominator);

    if xo < xm
      # The optimum is to the left of xm so make the upper estimate xm and make
      # the middle estimate xo.
      xu = xm;
      fxu = fxm;
      xm = xo;
      fxm = f(xm);
    elseif xo > xm
      # The optimum is to the right of xm so make the lower estimate xm and
      # the middle estimate xo.
      xl = xm;
      fxl = fxm;
      xm = xo;
      fxm = f(xm);
    else
      break;
    end

    # Relative error checks
    if abs((xo - xoold) / xo) * 100 < ea
      break;
    end
  end
  return xo;
end


module YaLinearAlgebra

import Base: +, -, *

export +, -, *, μ, dot, sum_of_square, norm,
  distance, shape, get_row, get_column, get_col, make_matrix

## Vector Portion

function +(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
  """
  Vector Additon is component-wise
  """
  @assert length(v₁) == length(v₂)
  v₁ .+ v₂
end

function +(vc::Vector{Vector{T}})::Vector{T} where T <: Number
  """
  Sum all corresponding components of a collection of vector to return
  one vector
  """
  @assert length(vc) > 0 "expect the collection to have at leat one vector element"
  n = length(vc[1])
  @assert all(v -> length(v) == n, vc) "Expect all vectors to be of he same length"
  v = zeros(T, n)
  for v₀ ∈ vc
    v .+= v₀
  end
  v
end

function -(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
  """
  Vector Additon is component-wise
  """
  @assert length(v₁) == length(v₂)
  v₁ .- v₂
end

*(x::T, v::Vector{T}) where T <: Number = v .* x
*(v::Vector{T}, x::T) where T <: Number = v .* x

function μ(vc::Vector{Vector{T}})::Vector{AbstractFloat} where T <: Number
  """Element-wise mean"""
  @assert length(vc) > 0
  n = length(vc[1])

  @assert all(v -> length(v) == n, vc)
  reduce(+, vc) / length(vc)
end

function dot(v₁::Vector{T}, v₂::Vector{T})::T where T <: Number
  @assert length(v₁) == length(v₂)
  sum(v₁ .* v₂)
end

function sum_of_square(v::Vector{T})::T where T <: Number
  """
  return vᵢ * vᵢ ∀ i ∈ 1:length(v)
  """
  dot(v, v)
end

function norm(v::Vector{T})::AbstractFloat where T <: Number
  √ sum(v .* v)
end

function distance(v₁::Vector{T}, v₂::Vector{T})::AbstractFloat where T <: Number
  """
  (Euclidean) distance(v₁, v₂) ≡ √(Σ (v₁ᵢ - v₂ᵢ)²)
  """
  @assert length(v₁) == length(v₂)
  √ sum((v₁ .- v₂).^2)  # == norm(v₁ .- v₂)
end

## Matrix Portion

function shape(m::Matrix{T})::Tuple{Integer, Integer} where T <: Number
  size(m)
end

function get_row(m::Matrix{T}, r::Integer)::Vector{T}  where T <: Number
  @assert 1 ≤ r ≤ size(m)[1]
  view(m, r, :)  ## no copy!
end

function get_column(m::Matrix{T}, c::Integer)::Vector{T}  where T <: Number
  @assert 1 ≤ c ≤ size(m)[2]
  view(m, :, c)  ## no copy!
end

get_col(m::Matrix{T}, c::Integer) where T <: Number = get_column(m, c)

function make_matrix(nrows::Integer, ncols::Integer, fn::Function;
    DT::DataType=Float64)::Matrix
  @assert 1 ≤ nrows && 1 ≤ ncols

  m = zeros(DT, (nrows, ncols))
  for c ∈ 1:ncols, r ∈ 1:nrows
    m[r, c] = fn(r, c)
  end
  m
end

end  ## YaLinearAlgebra

Data Science From Scratch/src/YaLinearAlgebra.jl


using LinearAlgebra
using SparseArrays
using Plots
using ForwardDiff

"This code performs a convergence test for two finite difference methods applied to
Laplace's equation with mixed Dirichlet-Neumann boundary conditions."

# method of manufactured solutions: define the exact solution, then figure out
# what f,α,β should be to make it the solution to your PDE
uexact(x) = log(2+sin(pi*x))
dudx_exact(x) = ForwardDiff.derivative(uexact,x)
f(x) = -ForwardDiff.derivative(dudx_exact,x) # -d^2u/dx^2 = f

α = uexact(-1)
β = dudx_exact(1)

# The O(h) accurate method
function solve1(m,f,α,β)
    x = LinRange(-1,1,m+2) # add 2 endpoints
    xint = x[2:end]
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
    A[end,end] = 1/h^2
    b = f.(xint)
    b[1] += α/h^2

    b[end] += β/h # approach 1 - O(h) accuracy only

    u = A\b
    return u,x,xint,h
end

# The O(h^2) accurate method
function solve2(m,f,α,β)
    x = LinRange(-1,1,m+2) # add 2 endpoints
    xint = x[2:end] # leave x_{m+1} in the list of points!
    h = x[2]-x[1]
    A = (1/h^2) * spdiagm(0=>2*ones(m+1),-1=>-ones(m),1=>-ones(m))
    A[end,end] = 1/h^2
    b = f.(xint)
    b[1] += α/h^2

    b[end] = .5*f(x[end]) + β/h

    u = A\b
    return u,x,xint,h
end

# # this code plots the solution
# m = 40
# u1,x,xint1,h = solve1(m,f,α,β)
# u2,x,xint2,h = solve2(m,f,α,β)
# plot(xint1,u1,mark=:dot,markersize=2,label="O(h) solution")
# plot!(xint2,u2,mark=:dot,markersize=2,label="O(h^2) solution")
# plot!(x,uexact.(x),label="Exact solution")
# plot!(leg=:bottomright)

mvec = 2 .^ (2:9)
hvec = zeros(length(mvec))
err1 = zeros(length(mvec))
err2 = zeros(length(mvec))
for (i,m) in enumerate(mvec)
    u1,x,xint,h = solve1(m,f,α,β)
    u2,x,xint,h = solve2(m,f,α,β)
    hvec[i] = h
    err1[i] = maximum(@. abs(uexact(xint) - u1))
    err2[i] = maximum(@. abs(uexact(xint) - u2))
end
plot(hvec,err1,marker=:dot,label="Max error for method 1")
plot!(hvec,err2,marker=:dot,label="Max error for method 2")
plot!(hvec,hvec.^2,linestyle=:dash,label="O(h^2)")
plot!(xaxis=:log,yaxis=:log,legend=:bottomright)


using LinearAlgebra

# k_local(Elastic Modulus,thickness,height,length,number of nodes,row/column one, row/column two)

#creates local stiffness matrix and maps to global
function k_local(k,l,n,rc_1,rc_2)
    k_global = zeros(n,n)
    k_global[rc_1,rc_1] = k
    k_global[rc_1,rc_2] = -k
    k_global[rc_2,rc_1] = -k
    k_global[rc_2,rc_2] = k
    return k_global
end

# constants from problem statement
E = 29*10^6
l = 2
k1 = 25
k2 = 35
k3 = 45
k4 = 30
# combining all globally mapped matrices
kG = k_local(k1,l,5,1,2)+
     k_local(k2,l,5,2,3)+
     k_local(k3,l,5,3,4)+
     k_local(k4,l,5,3,5)
kG[4,:] = Float64[0 0 0 1 0] #adjustment for Reaction from fixed end
kG[5,:] = Float64[0 0 0 0 1] #adjustment for Reaction from fixed end
kG[:,4] = transpose(Float64[0 0 0 1 0]) #adjustment for Reaction from fixed end
kG[:,5] = transpose(Float64[0 0 0 0 1]) #adjustment for Reaction from fixed end

# Force Vector
f = Float64[75 0 0 0 0]
F = transpose(f)

# Displacement vector
u = kG\F
println("Nodal displacement vector (inches):")
display(u)

# Strain Function
ϵ(x2,x1,l) = (x2 - x1)/l

#Strain Vector
ϵM = [ϵ(u[2],u[1],l)
      ϵ(u[3],u[2],l)
      ϵ(u[4],u[3],l)
      ϵ(u[5],u[3],l)]
println("Element strain vector (inch/inch):")
display(ϵM)

# Stress Function
σ(E,ϵ) = E*ϵ

# Stress Vector
σM = [σ(E,ϵM[1])
      σ(E,ϵM[2])
      σ(E,ϵM[3])
      σ(E,ϵM[4])]
println("Element Stress vector (psi):")
display(σM)

#Reaction Forces vector
Fapp(k,x2,x) = k*(x2-x)
FappM = [Fapp(k12,u[2],u[1])
         Fapp(k12,u[3],u[2])
         Fapp(k3,u[4],u[3])
         Fapp(k4,u[5],u[3])]
println("Reaction Forces vector (lbs):")
display(FappM)


"""
    univariate(k::Int64, l::Int64, low::N, high::N)::Vector{N} where {N<:AbstractFloat}

Compute the Bernstein coefficients of a univariate monomial.

### Input

- `k`    -- degree of the given monomial
- `l`    -- degree of the Bernstein polynomial
- `low`  -- the lower bound of the interval where the Bernstein coefficients are computed
- `high` -- the upper bound of the interval the Bernstein coefficients are computed

### Output

A vector with floating point entries containing the Bernstein coefficients.
"""
function univariate(k::Int64, l::Int64, low::N, high::N)::Vector{N} where {N<:AbstractFloat}
    m = l - k
    b = zeros(N, l+1)
    @inbounds @fastmath for i in 0:l
        if k < l
            for j in max(0, i-m):min(k, i)
                aux =  binomial(m, i-j) * binomial(k, j) / binomial(k+m, i)
                b[i+1] += aux * low^(k-j) * high^j
            end
        else
            b[i+1] = low^(k-i) * high^i
        end
    end
    return b
end

"""
    univariate(k::Int64, l::Int64, low::Rational, high::Rational)::Vector{Rational}

Compute *exactly* the Bernstein coefficients of a univariate monomial.

### Input

- `k`    -- degree of the given monomial
- `l`    -- degree of the Bernstein polynomial
- `low`  -- the lower bound of the interval where the Bernstein coefficients are computed
- `high` -- the upper bound of the interval the Bernstein coefficients are computed

### Output

A vector with rational entries containing the Bernstein coefficients.
"""
function univariate(k::Int64, l::Int64, low::Rational, high::Rational)::Vector{Rational}
    m = l - k
    b = zeros(Rational, l+1)
    @inbounds for i in 0:l
        if k < l
            for j in max(0, i-m):min(k, i)
                aux =  binomial(m, i-j) * binomial(k, j) // binomial(k+m, i)
                b[i+1] += aux * low^(k-j) * high^j
            end
        else
            b[i+1] = low^(k-i) * high^i
        end
    end
    return b
end

"""
    multivariate(k::Vector{Int64}, l::Vector{Int64},
                          low::Vector{N}, high::Vector{N})::ImplicitForm{N} where {N<:Number}

Compute the Bernstein coefficients of a multivariate monomial.

### Input

- `k`    -- vector of degrees for each monomial
- `l`    -- vector of degrees of the Bernstein polynomial for each monomial
- `low`  -- the lower bounds of the interval where the Bernstein coefficients are computed
- `high` -- the upper bounds of the interval the Bernstein coefficients are computed

### Output

A Bernstein implicit form holding the Bernstein coefficients.

### Examples

```julia
julia> m = multivariate([3,2],[3,2],[1.0,2],[2,4.0]);
julia> m.array
2-element Array{Array{Float64,1},1}:
 [1.0, 2.0, 4.0, 8.0]
 [4.0, 8.0, 16.0]
```
"""
function multivariate(k::Vector{Int64}, l::Vector{Int64},
                      low::Vector{N}, high::Vector{N})::ImplicitForm{N} where {N<:Number}
    n = length(low)
    ncoeffs = 1
    B = Vector{Vector{N}}(n)
    @inbounds for i in 1:n
        @views B[i] = univariate(k[i], l[i], low[i], high[i])
        ncoeffs = ncoeffs * length(B[i])
    end
    return ImplicitForm(B, n, ncoeffs)
end


using ForwardDiff, Roots, Plots, LinearAlgebra, Optim

function strong_backtracking(f, ∇, x, d; α=1, β=0.1, σ=0.4)
    y0, g0, y_prev, α_prev = f(x), ∇(x)⋅d, NaN, 0
    αlo, αhi = NaN, NaN
    # bracket phase
    while true
        y = f(x + α*d)
        if y > y0 + β*α*g0 || (!isnan(y_prev) && y ≥ y_prev)
            αlo, αhi = α_prev, α
            break
        end
        g = ∇(x + α*d)⋅d
        if abs(g) ≤ -σ*g0
            return α
        elseif g ≥ 0
            αlo, αhi = α, α_prev
            break
        end
        y_prev, α_prev, α = y, α, 2α
    end

    # zoom phase
    ylo = f(x + αlo*d)
    while true
        α = (αlo + αhi)/2
        y = f(x + α*d)
        if y > y0 + β*α*g0 || y ≥ ylo
            αhi = α
        else
            g = ∇(x + α*d)⋅d
            if abs(g) ≤ -σ*g0
                return α
            elseif g*(αhi - αlo) ≥ 0
                αhi = αlo
            end
            αlo = α
        end
    end
end

f(x) = ((x[1]-1.0)^2) + (0.1*(x[2]-4)^2) + sin(5*pi*x[1])+sin(5*pi*x[2])
grad_f(x) = ForwardDiff.gradient(f,x)

# Ponto inicial
xk = [-1.0,1.0]

#direcao de busca
dk = -grad_f(xk)/norm(grad_f(xk))

beta = 0.1
alpha_f = 1.0

phi(alpha) = f(xk+alpha*dk)
d_phi(alpha) = grad_f(xk+alpha*dk)'dk

l_alpha(alpha) = phi(0)+alpha*beta*d_phi(0)
sigma = 0.4

wolf = []

for alpha = 0:0.01:alpha_f
    if (phi(alpha) <= l_alpha(alpha)) && (abs(grad_f(xk+alpha*dk)'dk) <= -sigma*grad_f(xk)'dk)
        push!(wolf,alpha)
    end
end

wolfbt = strong_backtracking(f,grad_f,xk,dk)

plot(phi,0,alpha_f,lw=2,lab="phi")
plot!(l_alpha,0,alpha_f,lw=2,lab="L_phi",c=:black)
scatter!(wolf,phi.(wolf),lab="StrongWolfe",ms=3,c=:red)
scatter!([wolfbt],[phi(wolfbt)],lab="StrongWolf - BT",c=:yellow)
plot!(legend=:outerbottomright)
plot!(title="beta = 0.1 e sigma = 0.4",xlabel = "alpha", ylabel = "phi(alpha), l_alpha(alpha")


using SpecialMatrices
using LinearAlgebra
using SparseArrays

"""Ankara.jl contain the functions to develop the description of the
discrete and finite harmonic oscillator in terms of the Harper equations."""

# Δ1(j) gives the circulant matrix of size d_{j} × d_{j} where d_{j} = 2j + 1
function Δ1(j)
	Δ1 = zeros(Float64, 2j + 1)
	Δ1[1] = -2
	Δ1[2] = 1
	Δ1[end] = 1
	Δ1 = SpecialMatrices.Circulant(Δ1)
end

# Δ2(j) gives the diagonal matrix of size d_{j} × d_{j} whose diagonal entries
# are oscillating terms
function Δ2(j)
	Δ2 = zeros(Float64, 2j + 1)
	for i in -j:j
		Δ2[i + 1 + j] = -4 * (sin(pi * i / (2j)))^2
	end
	return Δ2 = LinearAlgebra.diagm(Δ2)
end

# If necessary, compose the Hamiltonian (Harper's equation) H from who the
# eigenvectors and eigenvalues are obtained
H(j) = -(1/2.0)*(Δ1(j) + Δ2(j))

# Eigenvectors of H
h(j) = LinearAlgebra.eigvecs(H(j));

# Eigenvalues of H. Altough, this eigvenvalues are not strictly used because
# of the symmetry importation below
k(j) = LinearAlgebra.eigvals(H(j));

# Lineal monotonic increasing values who serves as the (new) eigenvalues
# in the symmetry importation to construct the evolution operator
η(j) = [l for l in 0:(2j + 1)];

# Kernel of the evolution
ker(j, t) = exp.(-1im * pi * t * η(j)/2.0)


#' ## Run this cell to load the graphics packages
using Plots
gr()

#' ## Run this cell to use a couple of definitions from the previous
#' Tools notebook

sigma(x) = 1 / (1 + exp(-x))
f(x, w) = sigma(w * x)

#' ## Multiple function parameters
#'
#' In notebook 6, we saw how we could adjust a parameter to make a curve fit
#' a single data point. What if there is more data to fit?
#'
#' We'll see that as we acquire more data to fit, it can sometimes be useful
#' to add complexity to our model via additional parameters.

#' ## Multiple function parameters
#'
#' In notebook 6, we saw how we could adjust a parameter to make a curve fit
#' a single data point. What if there is more data to fit?
#'
#' We'll see that as we acquire more data to fit, it can sometimes be useful
#' to add complexity to our model via additional parameters.

#' ## Adding more data
#' Suppose there are now two data points to fit, the previous $(x_0, y_0)$
#' used in notebook 6 and also $(x_1, y_1) = (-3, 0.3)$.

#' #### Exercise 1
#
# Make an interactive plot of the function $f_w$ together with the two data
# points. Can you make the graph of $f_w$ pass through *both* data points
# at the *same* time?

x0, y0 = 2, 0.8
x1, y1 = -3, 0.3
w = 0.4
plot(x->f(x, w), -5, 5, ylims=(0, 1), label="f")
scatter!([x0,x1], [y0,y1], label="data")

#' You should have found that it's actually *impossible* to fit both data points
#' at the same time! The best we could do is to *minimise* how far away the
#' function is from the data. To do so, we need to somehow balance the distance
#' from each of the two data points.

#' #### Exercise 2
#'
#' Play with the slider to find the value $w^*$ of $w$ that you think has the "least error" in an intuitive sense.

w=0.45
plot(x->f(x, w), -5, 5, ylim=(0,1))
scatter!([x0, x1], [y0, y1])

#' ## Defining a loss function
#' How can we *quantify* some kind of overall measure of the distance from *all* of the data? Well, we just need to define a new loss function! One way to do so would be to sum up the loss functions for each data point, i.e. the sum of the squared vertical distances from the graph to each data point:
#'
#' $$L(w) = [y_0 - f_w(x_0)]^2 + [y_1 - f_w(x_1)]^2.$$
#'
#' Since the two pieces that we are adding have the same mathematical "shape" or structure, we can abbreviate this by writing it as a sum:
#'
#' $$L(w) = \sum_{i=0}^1 [y_i - f_w(x_i)]^2.$$

#' So now we want to find the value $w^*$ of $w$ that minimizes this new
#' function $L$!

#' #### Exercise 3
#'
#' Make a visualization to show the function $f_w$ and to visualize the
#' distance from each of the data points.
xs = [2, -3]
ys = [0.8, 0.3]

L(w) = sum( (ys .- f.(xs, w)) .^ 2 )

begin
    w = 0.3 # Try manipulating w between -2 and 2

    plot(x->f(x, w), -5, 5, ylims=(0, 1), lw = 3, label="f_w")

    scatter!(xs, ys, label="data")

    for i in 1:2
        plot!([xs[i], xs[i]], [ys[i], f(xs[i], w)], c=:green)
    end

    title!("L(w) =  $(round(L(w); sigdigits = 5))")
end

#' #### Exercise 4
#'
#' After playing with this for a while, it is intuitively obvious that we
# cannot make the function pass through both data points for any value of
# $w$. In other words, our loss function, `L(w)`, is never zero.
#'
#' What is the minimum value of `L(w)` that you can find by altering `w`?
#' What is the corresponding value of `w`?
w = 0.42
plot(x->f(x, w), -5, 5, ylims=(0, 1), lw = 3, label="f_w", legend=(0.8,0.1))

scatter!(xs, ys, label="data")

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], f(xs[i], w)], c=:green)
end

title!("L($w) =  $(round(L(w); sigdigits = 5))")

#' ### Sums in Julia
#' To generate the above plot we used the `sum` function.
#' `sum` can add together all the elements of a collection or range,
#' or it can add together the outputs of applying a function to all the
#' elements of a collection or range.
#'
#' Look up the docs for `sum` via
#'
#' ```julia
#' ?sum
#' ```
#' if you need more information.

#' #### Exercise 6
#'
#' What is the sum of the absolute values of all integers between -3 and 3? Use `sum` and the `abs` function.

#' #### Solution

sum(abs, -3:3)

#' ## What does the loss function $L$ look like?
#' In our last attempt to minimize `L(w)` by varying `w`, we saw that `L(w)`
#' always seemed to be greater than 0.  Is that true? Let's plot it to find out!

#' #### Exercise 7
#'
#' Plot the new loss function $L(w)$ as a function of $w$.
#' #### Solution
plot(L, -2, 2, xlabel="w", ylabel="L(w)", ylims=(0, 1.2))

#' ### Features of the loss function
#' The first thing to notice is that $L$ is always positive.
#' Since it is the sum of squares, and squares cannot be negative,
#' the sum cannot be negative either!
#'
#' However, we also see that although $L$ dips close to $0$ for a single,
#' special value $w^* \simeq 0.4$, it never actually *reaches* 0!
#' Again we could zoom in on that region of the graph to estimate it more
#' precisely.

#' We might start suspecting that there should be a better way of using the
#' computer to minimize $L$ to find the location $w^*$ of the minimum,
#' rather than still doing everything by eye. Indeed there is, as we will
#' see in the next two notebooks!

#' ## Adding more parameters to the model

#' If we add more parameters to a function, we may be able to improve how it
#' fits to data. For example, we could define a new function $g$ with another
#' parameter, a shift or **bias**:
#'
#' $$g(x; w, b) := \sigma(w \, x + b).$$
#+ results="hidden"
g(x, w, b) = sigma(w*x + b)
#+

xs = [2, -3]
ys = [0.8, 0.3]

L2D(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2)
w = 0.45 # Try manipulating w between -2 and 2
b = 0.5 # Try manipulating b between -2 and 2
plot(x -> g(x, w, b), -5, 5, ylims=(0, 1), lw=3)

scatter!(xs, ys)

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2D(w, b) = $(L2D(w, b))")

#' You should be able to convince yourself that we can now make the curve pass
#' through both points simultaneously.

#' #### Exercise 9
#'
#' For what values of `w` and `b` does the line pass through both points?

w = 0.45
b = 0.48
plot(x -> g(x, w, b), -5, 5, ylims=(0, 1), lw=3)

scatter!(xs, ys)

for i in 1:2
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2D(w, b) = $(L2D(w, b))")
#' ## Fitting both data points: a loss function
#' Following the procedure that we used when we had a single parameter,
#' we can think of the fitting procedure once again as minimizing a suitable
#' *loss function*. The expression for the loss function is almost the same,
#' except that now $f$ has two parameters $w$ and $b$, so the loss function
#' $L_2$ is itself a function of $w$ *and* $b$:
#'
#' $$L_2(w, b) = \sum_i [y_i - f(x_i; w, b)]^2.$$
#'
#' So we want to minimize this loss function over *both* of the parameters
#' $w$ and $b$! Let's plot it.

#' #### Exercise 10
#'
#' Define the function `L2` in Julia.

#' #### Solution

L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )

#' To plot the loss as a function of *two* variables (the weights *and* the bias),
#' we will make use of the `surface` function.  To get a nice interactive
# 3D plot, we will use the PlotlyJS "backend" (plotting engine):

gr()  # load the backend

#-

ws = -2:0.05:2
bs = -2:0.05:2

contour(ws, bs, L2, levels=0.001:0.025:1.25, xlabel="value of w", ylabel="value of b")

#' We can see that indeed there is a unique point $(w^*, b^*)$ where the
#' function $L_2$ attains its minimum. You can try different ranges for the
#' $x$, $y$ and $z$ coordinates to get a better view.

#' ## More data

#' If we add more data, however, we will again not be able to fit all of the
#' data; we will only be able to attain a "best fit".
#'
#' Let's create `xs` and `ys` with some more data:

xs = [2, -3, -1, 1]
ys = [0.8, 0.3, 0.4, 0.4]

#' #### Exercise 11
#'
#' a) Make an interactive visualization of the function $f$ and the data. Try
#' to find the values of $w$ and $b$ that give the best fit.
#'
#' b) Define the loss function and plot it.
#'
#' ```julia
#' sigma(x) = 1 / (1 + exp(-x))
#' f(x, w) = sigma(w * x)
#' g(x, w, b) = sigma(w*x + b)
#' ```
L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )

w = 0.4 # Try manipulating w between -2 and 2
b = -0.1 # Try manipulating b between -2 and 2
plot(x->g(x, w, b), -5, 5, ylims=(-0.2, 1))

scatter!(xs, ys)

for i in 1:length(xs)
    plot!([xs[i], xs[i]], [ys[i], g(xs[i], w, b)])
end

title!("L2(w, b) = $(round(L2(w, b); sigdigits = 5))")

#' Let's define the loss function that we're using above so that we can plot
#' it as a function of the parameters `w` and `b`.
#' #### Exercise 12
#'
#' We've seen the loss function, $$L_{2D}(w, b) = \sum_i[\mathrm{ys}_i - g(\mathrm{xs}_i, w, b)]^2,$$ written in Julia as
#
# ```julia
# L2(w, b) = sum( (ys .- g.(xs, w, b)) .^ 2 )
# ```
#
# a few times now. To ensure you understand what this function is doing, implement your own loss function using the commented code below. Do this without using `sum`, `abs2`, or broadcasting dot syntax (for example, `.-`). Hint: you'll want to use a `for` loop to do this.

function myL2(w, b)
    loss = 0.0
    for (x, y) in zip(xs, ys)
        loss += (y - g(x, w, b))^2
    end
    return loss
end

#' Now that you've defined `L2`, we can plot it using either `surface` or
#' `contour` to see how the loss changes as `w` and `b` change!

#' ## plotlyjs() # Use Plotly for 3d surface plots (if possible)

ws = -2:0.05:2
bs = -2:0.05:2

#' ## surface(ws, bs, L2, alpha=0.8, zlims=(0,3))  # alpha gives transparency
contour(ws, bs, L2, levels=0.055:0.05:1.5, xlabel="value of w", ylabel="value of b")


"""
    curve_length(f::Function,a::Real,b::Real;rtol=1e-13) -> Float64

Calculates the length of the curve given by `f`, where `f` is function of one parameter, on interval <`a`, `b`> of said parameter.

# Arguments
- `f::Function`: function of the curve to be measured,
- `a::Real`: left-bound of the interval to be integrated on,
- `b::Real`: right-bound of the interval to be integrated on,
- `rtol::Real`: default `rtol = 1e-13`, relative tolerance,

# Keywords

# Returns
- `Float64`: length of the curve defined by the `f` on the interval <`a`, `b`>.

# Throws
"""
function curve_length(f::Function,a::Real,b::Real;rtol=1e-13)
    df = x -> ForwardDiff.derivative(f,x)
    dr = x -> sqrt(1+df(x)^2)
    l,_ = quadgk(dr,a,b;rtol=rtol)
    return l
end

"""
    curve_fraction(f::Function,a::Real,b::Real,r::Real) -> Float64

Retunrs the position of the fraction of the length from the interval <`a`, `b`>, where the length of the curve is given by one parameter function `f`.

# Arguments
- `f::Function`: function of the curve,
- `a::Real`: left-bound of the interval to be integrated on,
- `b::Real`: right-bound of the interval to be integrated on,
- `r::Real`: fraction of the length of the curve given by `f` from the left,
- `rtol:Real`: default `rtol = 1e-13`, relative tolerance,

# Keywords

# Returns
- `Float64`: position of the fraction of the length from the interval <`a`, `b`>.

# Throws
"""
function curve_fraction(f::Function,a::Real,b::Real,r::Real;rtol=1e-13)
    df = x -> ForwardDiff.derivative(f,x)
    dr = x -> sqrt(1.0 + df(x)^2)

    l,_ = quadgk(dr,a,b;rtol=rtol)
    lf = l*r

    xo = (a+b)/2
    xn = 0.0
    eps_n = 1.0
    while eps_n > xo*rtol
        l,_ = quadgk(dr,a,xo;rtol=rtol)
        xn = xo - (l - lf)/dr(xo)
        eps_n = abs(xo-xn)
        xo = xn
    end

    return xo
end


###############################################################################
#
# Quadratic function solver
#
###############################################################################
"""

    lower_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}

Return the lower quadratic solution or NaN, given
- `a` Parameter in `a*x^2 + b*x + c = 0`
- `b` Parameter in `a*x^2 + b*x + c = 0`
- `c` Parameter in `a*x^2 + b*x + c = 0`

---
Example
```julia
low_x = lower_quadratic(1.0, 2.0, 0.0);
low_x = lower_quadratic(-1.0, 2.0, 0.0);
low_x = lower_quadratic(1.0, 2.0, 2.0);
```
"""
function lower_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}
    discr = b^2 - 4*a*c;

    if (discr >= 0) && (a > 0)
        return (-b - sqrt(discr)) / (2 * a)
    elseif (discr >= 0) && (a < 0)
        return (-b + sqrt(discr)) / (2 * a)
    else
        return FT(NaN)
    end
end

"""

    upper_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}

Return the upper quadratic solution or NaN, given
- `a` Parameter in `a*x^2 + b*x + c = 0`
- `b` Parameter in `a*x^2 + b*x + c = 0`
- `c` Parameter in `a*x^2 + b*x + c = 0`

---
Example
```julia
up_x = upper_quadratic(1.0, 2.0, 0.0);
up_x = upper_quadratic(-1.0, 2.0, 0.0);
up_x = upper_quadratic(1.0, 2.0, 2.0);
```
"""
function upper_quadratic(a::FT, b::FT, c::FT) where {FT<:AbstractFloat}
    discr = b^2 - 4*a*c;

    if (discr >= 0) && (a > 0)
        return (-b + sqrt(discr)) / (2 * a)
    elseif (discr >= 0) && (a < 0)
        return (-b - sqrt(discr)) / (2 * a)
    else
        return FT(NaN)
    end
end


using Optim, HTTP, GLM, LinearAlgebra, Random, Statistics, DataFrames, CSV, FreqTables
using Pkg
Pkg.add("Distributions")
using Pkg
Pkg.add("ForwardDiff")

    #Question1
    using DataFrames
using CSV
using HTTP
url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2021/master/ProblemSets/PS4-mixture/nlsw88t.csv"
df = CSV.read(HTTP.get(url).body, DataFrame)
X = [df.age df.white df.collgrad]
Z = hcat(df.elnwage1, df.elnwage2, df.elnwage3, df.elnwage4,
df.elnwage5, df.elnwage6, df.elnwage7, df.elnwage8)
y = df.occ_code
#Using Q1 from Pset3
function mlogit_with_Z(theta, X, Z, y)

    alpha = theta[1:end-1]
    gamma = theta[end]
    K = size(X,2)
    J = length(unique(y))
    N = length(y)
    bigY = zeros(N,J)
    for j=1:J
        bigY[:,j] = y.==j
    end
    bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

    T = promote_type(eltype(X),eltype(theta))
    num   = zeros(T,N,J)
    dem   = zeros(T,N)
    for j=1:J
        num[:,j] = exp.(X*bigAlpha[:,j] .+ (Z[:,j] .- Z[:,J])*gamma)
        dem .+= num[:,j]
    end

    P = num./repeat(dem,1,J)

    loglike = -sum( bigY.*log.(P) )

    return loglike
end
startvals = [2*rand(7*size(X,2)).-1; .1]
td = TwiceDifferentiable(theta -> mlogit_with_Z(theta, X, Z, y), startvals; autodiff = :forward)
# run the optimizer
theta_hat_optim_ad = optimize(td, startvals, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
theta_hat_mle_ad = theta_hat_optim_ad.minimizer
# evaluate the Hessian at the estimates
H  = Optim.hessian!(td, theta_hat_mle_ad)
theta_hat_mle_ad_se = sqrt.(diag(inv(H)))
println([theta_hat_mle_ad theta_hat_mle_ad_se]
#Question 2
#Interpretation of gamma#
#Question 3
n.
using Distributions
include("lgwt.jl") # make sure the function gets read in
# define distribution
d = Normal(0,1) # mean=0, standard deviation=1
e.
# get quadrature nodes and weights for 7 grid points
nodes, weights = lgwt(7,-4,4)
# now compute the integral over the density and verify it's 1
sum(weights.*pdf.(d,nodes))
# now compute the expectation and verify it's 0
sum(weights.*nodes.*pdf.(d,nodes))
#computing integrals using the above
d = Normal(0,2)
nodes, weights = lgwt(7,-5,5)
sum(weights.*(nodes.).*pdf.(d,nodes))

d = Normal(0,2)
nodes, weights = lgwt(10,-5,5)
sum(weights.*(nodes.).*pdf.(d,nodes))

#Question4