Related
I have the following code in which the JuMP constraint is throwing an error.
using JuMP
using MosekTools
K = 3
N = 2
penalties = [1.0, 3.9, 8.7]
function A_tau(r::Number, n::Number, tau::Float64)
fac = 1
for m in 1:r
fac *= (n - (m - 1))
end
if n >= r
return fac * tau ^ (n - r)
else
return 0.0
end
end
function A_tau_mat(tau::Float64)
mat = Array{Float64, 2}(undef, N+1, N+1)
for i in 1:N+1
for j in 1:N+1
mat[i, j] = A_tau(i, j, tau)
end
end
return mat
end
m = Model(optimizer_with_attributes(Mosek.Optimizer, "QUIET" => false, "INTPNT_CO_TOL_DFEAS" => 1e-7))
#variable(m, p[1:1:K,1:1:N+1])
#variable(m, A[1:1:K+1,1:1:K,1:1:N+1,1:1:N+1])
#constraint(m, -A_tau_mat(0.0) * p[1, :] == [0.0, 0.0, 0.0])
optimize!(m)
println("p value is ", value.(p[1, :]))
println(A_tau_mat(0.0))
The error happens when the #constraint line is added and there's no error without it. The error is as follows. The error shows no method matching CartesianIndex ::Int64.
ERROR: LoadError: MethodError: no method matching -(::CartesianIndex{1}, ::Int64)
Closest candidates are:
-(!Matched::Complex{Bool}, ::Real) at complex.jl:307
-(!Matched::Missing, ::Number) at missing.jl:115
-(!Matched::MutableArithmetics.Zero, ::Any) at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/rewrite.jl:55
...
Stacktrace:
[1] _add_mul_array(::Array{GenericAffExpr{Float64,VariableRef},1}, ::Array{Float64,2}, ::JuMP.Containers.DenseAxisArray{VariableRef,1,Tuple{StepRange{Int64,Int64}},Tuple{Dict{Int64,Int64}}}) at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/linear_algebra.jl:166
[2] mutable_operate! at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/linear_algebra.jl:196 [inlined]
[3] mutable_operate_to!(::Array{GenericAffExpr{Float64,VariableRef},1}, ::typeof(*), ::Array{Float64,2}, ::JuMP.Containers.DenseAxisArray{VariableRef,1,Tuple{StepRange{Int64,Int64}},Tuple{Dict{Int64,Int64}}}) at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/linear_algebra.jl:208
[4] operate at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/linear_algebra.jl:221 [inlined]
[5] operate at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/rewrite.jl:43 [inlined]
[6] operate_fallback! at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/interface.jl:275 [inlined]
[7] operate!(::typeof(MutableArithmetics.sub_mul), ::MutableArithmetics.Zero, ::Array{Float64,2}, ::JuMP.Containers.DenseAxisArray{VariableRef,1,Tuple{StepRange{Int64,Int64}},Tuple{Dict{Int64,Int64}}}) at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/rewrite.jl:70
[8] top-level scope at /Users/prikshetsharma/.julia/packages/MutableArithmetics/0tlz5/src/rewrite.jl:227
[9] top-level scope at /Users/prikshetsharma/.julia/packages/JuMP/qhoVb/src/macros.jl:440
[10] top-level scope at /Users/prikshetsharma/Documents/clotorch/src/clotorch/flight/trajectory.jl:72
[11] include(::Function, ::Module, ::String) at ./Base.jl:380
[12] include(::Module, ::String) at ./Base.jl:368
[13] exec_options(::Base.JLOptions) at ./client.jl:296
[14] _start() at ./client.jl:506
in expression starting at /Users/prikshetsharma/Documents/clotorch/src/clotorch/flight/trajectory.jl:72
How to fix this error and use the constraint like I want to? What's wrong with this constraint?
You can try this
#variable(m, p[1:K,1:N+1])
#variable(m, A[1:K+1,1:K,1:N+1,1:N+1])
#constraint(m, -A_tau_mat(0.0) * p[1, :] .== [0.0, 0.0, 0.0])
There are two problems. First, the p's type in your original is DenseAxisArray rather than the normal Array because a StepRange (1:1:3) rather than a UnitRange (1:3) is provided. Though having the same elements, their types are different and the matrix multiplication implementation for it is somewhat problematic. I think it is a bug that should be fixed on the JuMP side. The other change is the dot . before the ==, which indicates broadcasting.
I'm trying to learn Julia via project Euler. And for the largest prime factor problem I ran into an issue with out of memory error and I don't understand why.
I have tried the same solution in Python and had no issues.
function Primes(n)
numbers = Set{Int64}(2:n)
primes = Int64[]
while length(numbers)!=0
p = pop!(numbers,minimum(numbers))
push!(primes,p)
if length(numbers)!=0
numbers = setdiff(numbers,Set{Int64}(2*p:p:maximum(numbers)))
end
end
return primes
end
function LPF(n)
primes = Primes(n//2)
for p in primes
if n%p==0
return p
end
end
end
LPF(600851475143)
this is my error message:
OutOfMemoryError()
Stacktrace:
[1] _growend! at .\array.jl:811 [inlined]
[2] resize! at .\array.jl:1003 [inlined]
[3] rehash!(::Dict{Int64,Nothing}, ::Int64) at .\dict.jl:183
[4] sizehint! at .\dict.jl:242 [inlined]
[5] sizehint! at .\set.jl:64 [inlined]
[6] union!(::Set{Int64}, ::UnitRange{Rational{Int64}}) at .\abstractset.jl:79
[7] Type at .\set.jl:10 [inlined]
[8] Primes(::Rational{Int64}) at .\In[14]:2
[9] LPF(::Int64) at .\In[24]:2
[10] top-level scope at In[25]:1
I am having trouble with pmap() throwing a BoundsError when setting the values of array elements - my code works for 1 worker but not >1. I have written a minimum working example which roughly follows the real code flow:
Get source data
Define set of points over which to iterate
Initialise array points to be calculated
Calculate each array point
The main file:
#pmapdemo.jl
using Distributed
#addprocs(length(Sys.cpu_info())) # uncomment this line for error
#everywhere include(joinpath(#__DIR__, "pmapdemo2.jl"))
function main()
# Get source data
source = Dict{String, Any}("t"=>zeros(5),
"x"=>zeros(5,6),
"y"=>zeros(5,3),
"z"=>zeros(5,3))
# Define set of points over which to iterate
iterset = Dict{String, Any}("t"=>source["t"],
"x"=>source["x"],
"y"=>fill(2, size(source["t"])[1], 1),
"z"=>fill(2, size(source["t"])[1], 1))
data = Dict{String, Any}()
# Initialise array points to be calculated
MyMod.initialisearray!(data, iterset)
# Calculate each array point
MyMod.calcarray!(data, iterset, source)
#show data
end
main()
The functionality file:
#pmapdemo2.jl
module MyMod
using Distributed
#everywhere using SharedArrays
# Initialise data array
function initialisearray!(data, fieldset)
zerofield::SharedArray{Float64, 4} = zeros(size(fieldset["t"])[1],
size(fieldset["x"])[2],
size(fieldset["y"])[2],
size(fieldset["z"])[2])
data["field"] = deepcopy(zerofield)
end
# Calculate values of array elements according to values in source
function calcpoint!((data, source, a, b, c, d))
data["field"][a,b,c,d] = rand()
end
# Set values in array
function calcarray!(data, iterset, source)
for a in eachindex(iterset["t"])
# [additional functionality f(a) here]
b = eachindex(iterset["x"][a,:])
c = eachindex(iterset["y"][a,:])
d = eachindex(iterset["z"][a,:])
pmap(calcpoint!, Iterators.product(Iterators.repeated(data,1), Iterators.repeated(source,1), Iterators.repeated(a,1), b, c, d))
end
end
end
The error output:
ERROR: LoadError: On worker 2:
BoundsError: attempt to access 0×0×0×0 Array{Float64,4} at index [1]
setindex! at ./array.jl:767 [inlined]
setindex! at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/SharedArrays/src/SharedArrays.jl:500 [inlined]
_setindex! at ./abstractarray.jl:1043
setindex! at ./abstractarray.jl:1020
calcpoint! at /home/dave/pmapdemo2.jl:25
#112 at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269
run_work_thunk at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:56
macro expansion at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269 [inlined]
#111 at ./task.jl:259
Stacktrace:
[1] (::getfield(Base, Symbol("##696#698")))(::Task) at ./asyncmap.jl:178
[2] foreach(::getfield(Base, Symbol("##696#698")), ::Array{Any,1}) at ./abstractarray.jl:1866
[3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at ./asyncmap.jl:178
[4] #async_usemap#681 at ./asyncmap.jl:154 [inlined]
[5] #async_usemap at ./none:0 [inlined]
[6] #asyncmap#680 at ./asyncmap.jl:81 [inlined]
[7] #asyncmap at ./none:0 [inlined]
[8] #pmap#213(::Bool, ::Int64, ::Nothing, ::Array{Any,1}, ::Nothing, ::Function, ::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:126
[9] pmap(::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:101
[10] #pmap#223(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
[11] pmap(::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
[12] calcarray!(::Dict{String,Any}, ::Dict{String,Any}, ::Dict{String,Any}) at /home/dave/pmapdemo2.jl:20
[13] main() at /home/dave/pmapdemo.jl:19
[14] top-level scope at none:0
in expression starting at /home/dave/pmapdemo.jl:23
In pmapdemo2.jl, replacing data["field"][a,b,c,d] = rand() with #show a, b, c, d demonstrates that all workers are running and have full access to the variables being passed, however instead replacing it with #show data["field"] throws the same error. Surely the entire purpose of SharedArrays is to avoid this? Or am I misunderstanding how to use it with pmap?
This is a crosspost from the Julia discourse here.
pmap will do the work of passing the data to the processes, so you don't need to use SharedArrays. Typically, the function provided to pmap (and indeed map) will be a pure function (and therefore doesn't mutate any variable) which returns one element of an output array. That function is mapped across each element of the input array, and the pmap function will construct the output array for you. For example, in your case, the code may look a bit like this
calcpoint(source, (a,b,c,d)) = rand() # Or some function of source and the indices a,b,c,d
field["data"] = pmap(calcpoint, Iterators.repeated(source), Iterators.product(a,b,c,d))
Dear users of the language julia. I have a problem when using the optimize function of the Optim package. What is the error of the code below?
using Optim
using Distributions
rng = MersenneTwister(1234);
d = Weibull(1,1)
x = rand(d,1000)
function pdf_weibull(x, lambda, k)
k/lambda * (x/lambda).^(k-1) * exp((-x/lambda)^k)
end
function obj(x::Vector, lambda, k)
soma = 0
for i in x
soma = soma + log(pdf_weibull(i,lambda,k))
end
-soma
end
obj(x, pars) = obj(x, pars...)
optimize(vars -> obj(x, vars...), [1.0,1.0])
Output
julia> optimize(vars -> obj(x, vars...), [1.0,1.0])
ERROR: DomainError:
Exponentiation yielding a complex result requires a complex argument.
Replace x^y with (x+0im)^y, Complex(x)^y, or similar.
Stacktrace:
[1] nan_dom_err at ./math.jl:300 [inlined]
[2] ^ at ./math.jl:699 [inlined]
[3] (::##2#4)(::Float64, ::Float64, ::Float64) at ./<missing>:0
[4] pdf_weibull(::Float64, ::Float64, ::Float64) at ./REPL[6]:2
[5] obj(::Array{Float64,1}, ::Float64, ::Float64) at ./REPL[7]:4
[6] (::##5#6)(::Array{Float64,1}) at ./REPL[11]:1
[7] value(::NLSolversBase.NonDifferentiable{Float64,Array{Float64,1},Val{false}}, ::Array{Float64,1}) at /home/pedro/.julia/v0.6/NLSolversBase/src/interface.jl:19
[8] initial_state(::Optim.NelderMead{Optim.AffineSimplexer,Optim.AdaptiveParameters}, ::Optim.Options{Float64,Void}, ::NLSolversBase.NonDifferentiable{Float64,Array{Float64,1},Val{false}}, ::Array{Float64,1}) at /home/pedro/.julia/v0.6/Optim/src/multivariate/solvers/zeroth_order/nelder_mead.jl:139
[9] optimize(::NLSolversBase.NonDifferentiable{Float64,Array{Float64,1},Val{false}}, ::Array{Float64,1}, ::Optim.NelderMead{Optim.AffineSimplexer,Optim.AdaptiveParameters}, ::Optim.Options{Float64,Void}) at /home/pedro/.julia/v0.6/Optim/src/multivariate/optimize/optimize.jl:25
[10] #optimize#151(::Array{Any,1}, ::Function, ::Tuple{##5#6}, ::Array{Float64,1}) at /home/pedro/.julia/v0.6/Optim/src/multivariate/optimize/interface.jl:62
[11] #optimize#148(::Array{Any,1}, ::Function, ::Function, ::Array{Float64,1}) at /home/pedro/.julia/v0.6/Optim/src/multivariate/optimize/interface.jl:52
[12] optimize(::Function, ::Array{Float64,1}) at /home/pedro/.julia/v0.6/Optim/src/multivariate/optimize/interface.jl:52
[13] macro expansion at ./REPL.jl:97 [inlined]
[14] (::Base.REPL.##1#2{Base.REPL.REPLBackend})() at ./event.jl:73
It is a simple problem to obtain the maximum likelihood estimates of the parameters that index the weibull distribution.
Best regards.
The reason of your problem is that your definition of pdf_weibull is incorrect. Here is a corrected definition:
function pdf_weibull(x, lambda, k)
k/lambda * (x/lambda)^(k-1) * exp(-(x/lambda)^k)
end
Note that I have moved - sign in exp part of the expression. If you change this all will work as expected.
Now - why does Julia complain with DomainError. The reason is that because of the error in your code you try to calculate the value of something like (-1.0)^0.5. In Julia ^ is implemented in type stable way. This means, in particular, that when it is passed Float64 as both arguments it guarantees to return Float64 or throw an error. Clearly (-1.0)^0.5 cannot be computed in real domain - that is why an error is thrown. If you passed (-1+0im)^0.5 then there would be no error as we are passing a complex number to ^ so the result can also be complex number in a type stable way.
Introduction
I have a directory with the following structure
--> Report
--> Problems
--> PE_001
--> Julia
PE_001.naive.jl
PE_001.jl
--> Benchmarks
test_001.txt
test_002.txt
--> Results
--> PE_002
.
.
.
--> PE_XXX
--> Benchmark
I am attempting to iterate over all the Julia files and benchmark them against the benchmarking data located under the top directory Benchmark. I do not want to have to cd into each directory and run # belapsed from the julia commandline to time every function individually.
To solve this problem I wrote the following code that is supposed to be located under benchmarks in the hierachy above. However I made it slightly simpler for illustrative purposes.
Attempt at solution
EDIT: The code below does NOT follow the hierarchy outlined above. To quickly reproduce the error, the code below has been written in such a way that all the files can be placed in the same directory.
benchmark.jl
include("PE_002.jl")
using BenchmarkTools
function get_file_path(PE=1)
current_folder = pwd()
PE_folder = "/PE_" * lpad(string(PE),3,"0")
dirname(pwd()) * "/Problems" * PE_folder * "/Julia"
end
function include_files(PE_dir)
for filename in readdir(PE_dir)
if !startswith(filename, "benchmark")
filepath = PE_dir * "/" * filename
#everywhere include($filepath)
end
end
end
function benchmark_files(PE_dir)
for filename in readdir(PE_dir)
if !startswith(filename, "benchmark")
f = getfield(Main, Symbol(filename[1:end-3]))
# Produces an error
println(#belapsed f())
end
end
end
# Works
println(#belapsed PE_002())
PE_dir = pwd()
include_files(PE_dir)
benchmark_files(PE_dir)
PE_002.jl
function PE_002(limit = 4*10^6)
a, b = 0, 2
while b < limit
a, b = b, 4 * b + a
end
div(a + b - 2, 4)
end
PE_002_naive.jl
function PE_002_naive(limit=4 * 10^6, F_1=1, F_2=2)
total = 0
while F_2 < limit
if F_2 % 2 == 0
total += F_2
end
F_1, F_2 = F_2, F_1 + F_2
end
total
end
test_001.txt
0*10**(2**0)
4*10**(2**0)
4*10**(2**1)
4*10**(2**2)
4*10**(2**3)
4*10**(2**4)
4*10**(2**5)
4*10**(2**6)
Question
Interestingly enough including the file PE_002 and then running # belapsed works, however obtaining the filename from the directory, turning it into a symbol, then trying to time it with #belapsed fails.
I know #elapsed works, however, due to garbage collection it is not nearly accurate enough for my needs.
Is there a simple way to benchmark all files in a remote directory using BenchmarkTools or similar tools as accurate?
All I need is a single number representing mean / average running time from each file.
EDIT 2: Per request I have included the full error message below
~/P/M/Julia-belaps ❯❯❯ julia benchmark.jl
9.495495495495496e-9
ERROR: LoadError: UndefVarError: f not defined
Stacktrace:
[1] ##core#665() at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:290
[2] ##sample#666(::BenchmarkTools.Parameters) at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:296
[3] #_run#6(::Bool, ::String, ::Array{Any,1}, ::Function, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:324
[4] (::BenchmarkTools.#kw##_run)(::Array{Any,1}, ::BenchmarkTools.#_run, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at ./<missing>:0
[5] anonymous at ./<missing>:?
[6] #run_result#16(::Array{Any,1}, ::Function, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:40
[7] (::BenchmarkTools.#kw##run_result)(::Array{Any,1}, ::BenchmarkTools.#run_result, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at ./<missing>:0
[8] #run#17(::Array{Any,1}, ::Function, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:43
[9] (::Base.#kw##run)(::Array{Any,1}, ::Base.#run, ::BenchmarkTools.Benchmark{Symbol("##benchmark#664")}, ::BenchmarkTools.Parameters) at ./<missing>:0 (repeats 2 times)
[10] macro expansion at /home/oisov/.julia/v0.6/BenchmarkTools/src/execution.jl:208 [inlined]
[11] benchmark_files(::String) at /home/oisov/Programming/Misc/Julia-belaps/benchmark.jl:26
[12] include_from_node1(::String) at ./loading.jl:569
[13] include(::String) at ./sysimg.jl:14
[14] process_options(::Base.JLOptions) at ./client.jl:305
[15] _start() at ./client.jl:371
while loading /home/oisov/Programming/Misc/Julia-belaps/benchmark.jl, in expression starting on line 36
Change println(#belapsed f()) to println(#belapsed $f()). I can't fully explain it, but here is a link to the relevant part of the docs.