How to calculate the runtime of MPI program in mpi4py - mpi

I wrote this code and tried some functions to calculate the time but it shows an error while running.I tried to use the C functions that is used to calculate runtime of each processor by using Wtime() and MPI_Reduce.Wtime() works I guess but the reduce function does not work the way it should.
The code is
from mpi4py import MPI
import mpi4py
import numpy as np
comm = MPI.COMM_WORLD
size = comm.size
rank = comm.Get_rank()
time1 = mpi4py.MPI.Wtime()
a = np.random.randint(10, size=(10, 10))
if rank == 0:
b = np.random.randint(10, size=(10, 10))
print(b)
else:
b = None
b = comm.bcast(b, root=0)
c = np.dot(a, b)
if size == 1:
result = np.dot(a, b)
else:
if rank == 0:
a_row = a.shape[0]
if a_row >= size:
split = np.array_split(a, size, axis=0)
else:
split = None
split = comm.scatter(split, root=0)
split = np.dot(split, b)
data = comm.gather(split, root=0)
time2 = mpi4py.MPI.Wtime()
duration = time2 - time1
totaltime = comm.reduce(duration,op = sum, root = 0)
print("Runtime at %d is %f" %(rank,duration))
if rank == 0:
result = np.vstack(data)
print(result)
print(totaltime)
The error it shows is
Runtime at 3 is 0.000574
Traceback (most recent call last):
File "matrixmultMPI.py", line 48, in <module>
totaltime = comm.reduce(duration,op = sum, root = 0)
File "mpi4py/MPI/Comm.pyx", line 1613, in mpi4py.MPI.Comm.reduce
File "mpi4py/MPI/msgpickle.pxi", line 1322, in mpi4py.MPI.PyMPI_reduce
File "mpi4py/MPI/msgpickle.pxi", line 1254, in mpi4py.MPI.PyMPI_reduce_intra
File "mpi4py/MPI/msgpickle.pxi", line 1126, in mpi4py.MPI.PyMPI_reduce_p2p
TypeError: 'float' object is not iterable
Traceback (most recent call last):
File "matrixmultMPI.py", line 48, in <module>
totaltime = comm.reduce(duration,op = sum, root = 0)
File "mpi4py/MPI/Comm.pyx", line 1613, in mpi4py.MPI.Comm.reduce
File "mpi4py/MPI/msgpickle.pxi", line 1322, in mpi4py.MPI.PyMPI_reduce
File "mpi4py/MPI/msgpickle.pxi", line 1254, in mpi4py.MPI.PyMPI_reduce_intra
File "mpi4py/MPI/msgpickle.pxi", line 1126, in mpi4py.MPI.PyMPI_reduce_p2p
TypeError: 'float' object is not iterable
Traceback (most recent call last):
File "matrixmultMPI.py", line 48, in <module>
totaltime = comm.reduce(duration,op = sum, root = 0)
File "mpi4py/MPI/Comm.pyx", line 1613, in mpi4py.MPI.Comm.reduce
File "mpi4py/MPI/msgpickle.pxi", line 1322, in mpi4py.MPI.PyMPI_reduce
File "mpi4py/MPI/msgpickle.pxi", line 1254, in mpi4py.MPI.PyMPI_reduce_intra
File "mpi4py/MPI/msgpickle.pxi", line 1126, in mpi4py.MPI.PyMPI_reduce_p2p
TypeError: 'float' object is not iterable
What's wrong with it and how do I go about it!!?

Related

In PyCaret getting error:"ValueError: Cannot cast object dtype to float64"

While using PyCaret's ML capabilities I am facing the following error.
The ML Code I am using:
if choice == "ML":
st.title("Your Machine Learning Process Starts Here!")
target = st.selectbox("Select Your Target", df.columns)
setup(df, target=target, silent=True)
setup_df = pull()
st.info("This is the ML Experiment Settings")
st.dataframe(setup_df)
best_model = compare_models()
compare_df = pull()
st.info("This is the ML Model")
st.dataframe(compare_df)
best_model
Now while running the code I am getting the error:
TypeError: setup() got an unexpected keyword argument 'silent'
Traceback:
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\streamlit\runtime\scriptrunner\script_runner.py", line 563, in _run_script
exec(code, module.__dict__)
File "D:\Python Projects\Machine-Learning-App\app.py", line 37, in <module>
setup(df, target=target, silent=True)
Hence, I removed the silent=True and now am getting the following error:
ValueError: Cannot cast object dtype to float64
Traceback:
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\streamlit\runtime\scriptrunner\script_runner.py", line 563, in _run_script
exec(code, module.__dict__)
File "D:\Python Projects\Machine-Learning-App\app.py", line 37, in <module>
setup(df, target=target)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\classification\functional.py", line 596, in setup
return exp.setup(
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\classification\oop.py", line 885, in setup
self.pipeline.fit(self.X_train, self.y_train)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\internal\pipeline.py", line 211, in fit
X, y, _ = self._fit(X, y, **fit_params_steps)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\internal\pipeline.py", line 192, in _fit
X, y, fitted_transformer = self._memory_fit(
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\joblib\memory.py", line 594, in __call__
return self._cached_call(args, kwargs)[0]
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\joblib\memory.py", line 537, in _cached_call
out, metadata = self.call(*args, **kwargs)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\joblib\memory.py", line 779, in call
output = self.func(*args, **kwargs)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\internal\pipeline.py", line 87, in _fit_transform_one
_fit_one(transformer, X, y, message, **fit_params)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\internal\pipeline.py", line 54, in _fit_one
transformer.fit(*args, **fit_params)
File "C:\Users\aviparna.biswas\AppData\Roaming\Python\Python39\site-packages\pycaret\internal\preprocess\transformers.py", line 216, in fit
self.transformer.fit(*args, **fit_params)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\sklearn\impute\_base.py", line 364, in fit
X = self._validate_input(X, in_fit=True)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\sklearn\impute\_base.py", line 319, in _validate_input
raise ve
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\sklearn\impute\_base.py", line 302, in _validate_input
X = self._validate_data(
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\sklearn\base.py", line 577, in _validate_data
X = check_array(X, input_name="X", **check_params)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\sklearn\utils\validation.py", line 791, in check_array
array = array.astype(new_dtype)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\generic.py", line 5912, in astype
new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\internals\managers.py", line 419, in astype
return self.apply("astype", dtype=dtype, copy=copy, errors=errors)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\internals\managers.py", line 304, in apply
applied = getattr(b, f)(**kwargs)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\internals\blocks.py", line 580, in astype
new_values = astype_array_safe(values, dtype, copy=copy, errors=errors)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\dtypes\cast.py", line 1292, in astype_array_safe
new_values = astype_array(values, dtype, copy=copy)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\dtypes\cast.py", line 1234, in astype_array
values = values.astype(dtype, copy=copy)
File "D:\Python Projects\Machine-Learning-App\mlapp\lib\site-packages\pandas\core\arrays\categorical.py", line 556, in astype
raise ValueError(msg)
Is there any workaround this error?
For installing pycaret I have used the command pip install -U --pre pycaret the latest release 3.0.0rc2 which works in python 3.9.
I had the same issue. It is working for me when I use python 3.7.9, but not 3.8.x or 3.9.x (as of now).
The 3.0.0rc2 version does not support silent parameter in setup() anymore. Hence the first error.
About this line:
setup(df, target=target)
The error would indicate that your labels are not of numerical type but an object. I'd look into that.
This is due to Pycaret not being compatible with scikit-learn version.
As the documentation says (https://pycaret.gitbook.io/docs/get-started/installation)
"PyCaret is not yet compatible with sklearn>=0.23.2"
Try downgrading scikit-learn:
pip install scikit-learn==0.23.2

How do I use Gather and Scatter functions in MPI using python mpi4py library for matrix multiplication

I was writing the code for Matrix Multiplication in python using mpi4py library and I am stuck on the part where Gather and Scatter functions are used, there is not much documentation regarding how to use it. The only websites I was able to find was this one which has the Official documentation about the library which is very new and the other one was this page.
The code I have written so far is
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank=comm.rank
size=comm.size
a = [[12,7,3],
[4 ,5,6],
[7 ,8,9]]
b = [[5,8,1],
[6,7,3],
[4,5,9]]
res = [[0,0,0],
[0,0,0],
[0,0,0]]
aa = [0,0,0]
cc = [0,0,0]
sum = 0
comm.Scatter(a, aa, root = 0)
for r in range(size):
if rank == r:
print("[%d] %s" %(rank, aa)) #this was just to check whether the Scattered values are being sent or not
comm.Bcast(b,root = 0)
for i in range(len(a)):
for j in range(len(b[0])):
for k in range(len(b)):
sum = sum + aa[j] * b[j][i]
cc[i] = sum #main calculation
comm.Gather(cc, res, root = 0) #here it should gather all values from cc to res matrix
if rank == 0:
for r in res:
print(r) #final printing of the res matrix
The error it's showing is
Traceback (most recent call last):
File "matrixmultMPI.py", line 22, in <module>
comm.Scatter(a, aa, root = 0)
File "mpi4py/MPI/Comm.pyx", line 740, in mpi4py.MPI.Comm.Scatter
File "mpi4py/MPI/msgbuffer.pxi", line 606, in mpi4py.MPI._p_msg_cco.for_scatter
File "mpi4py/MPI/msgbuffer.pxi", line 511, in mpi4py.MPI._p_msg_cco.for_cco_recv
File "mpi4py/MPI/msgbuffer.pxi", line 203, in mpi4py.MPI.message_simple
File "mpi4py/MPI/msgbuffer.pxi", line 138, in mpi4py.MPI.message_basic
File "mpi4py/MPI/asbuffer.pxi", line 365, in mpi4py.MPI.getbuffer
File "mpi4py/MPI/asbuffer.pxi", line 148, in mpi4py.MPI.PyMPI_GetBuffer
File "mpi4py/MPI/asbuffer.pxi", line 140, in mpi4py.MPI.PyMPI_GetBuffer
TypeError: a bytes-like object is required, not 'int'
Traceback (most recent call last):
File "matrixmultMPI.py", line 22, in <module>
comm.Scatter(a, aa, root = 0)
File "mpi4py/MPI/Comm.pyx", line 740, in mpi4py.MPI.Comm.Scatter
File "mpi4py/MPI/msgbuffer.pxi", line 606, in mpi4py.MPI._p_msg_cco.for_scatter
File "mpi4py/MPI/msgbuffer.pxi", line 511, in mpi4py.MPI._p_msg_cco.for_cco_recv
File "mpi4py/MPI/msgbuffer.pxi", line 203, in mpi4py.MPI.message_simple
File "mpi4py/MPI/msgbuffer.pxi", line 138, in mpi4py.MPI.message_basic
File "mpi4py/MPI/asbuffer.pxi", line 365, in mpi4py.MPI.getbuffer
File "mpi4py/MPI/asbuffer.pxi", line 148, in mpi4py.MPI.PyMPI_GetBuffer
File "mpi4py/MPI/asbuffer.pxi", line 140, in mpi4py.MPI.PyMPI_GetBuffer
TypeError: a bytes-like object is required, not 'int'
Traceback (most recent call last):
File "matrixmultMPI.py", line 22, in <module>
comm.Scatter(a, aa, root = 0)
File "mpi4py/MPI/Comm.pyx", line 740, in mpi4py.MPI.Comm.Scatter
File "mpi4py/MPI/msgbuffer.pxi", line 606, in mpi4py.MPI._p_msg_cco.for_scatter
File "mpi4py/MPI/msgbuffer.pxi", line 511, in mpi4py.MPI._p_msg_cco.for_cco_recv
File "mpi4py/MPI/msgbuffer.pxi", line 203, in mpi4py.MPI.message_simple
File "mpi4py/MPI/msgbuffer.pxi", line 138, in mpi4py.MPI.message_basic
File "mpi4py/MPI/asbuffer.pxi", line 365, in mpi4py.MPI.getbuffer
File "mpi4py/MPI/asbuffer.pxi", line 148, in mpi4py.MPI.PyMPI_GetBuffer
File "mpi4py/MPI/asbuffer.pxi", line 140, in mpi4py.MPI.PyMPI_GetBuffer
TypeError: a bytes-like object is required, not 'int'
Traceback (most recent call last):
File "matrixmultMPI.py", line 22, in <module>
comm.Scatter(a, aa, root = 0)
File "mpi4py/MPI/Comm.pyx", line 740, in mpi4py.MPI.Comm.Scatter
File "mpi4py/MPI/msgbuffer.pxi", line 597, in mpi4py.MPI._p_msg_cco.for_scatter
File "mpi4py/MPI/msgbuffer.pxi", line 495, in mpi4py.MPI._p_msg_cco.for_cco_send
File "mpi4py/MPI/msgbuffer.pxi", line 186, in mpi4py.MPI.message_simple
ValueError: too many values to unpack (expected 2)
Any help would be appreciated.

How to measure the length of a call stack?

Recently had a test question asking "how deep" the call stack for fact1 where n = 5. Here is the code:
int fact1(int n)
{
if (n == 1)
{
return 1
}
else {
return n * fact(n-1)
}
}
The answer on the test was 5, but I believe it is 4. I don't believe the first call is to be counted in the number of calls.
Actually, every function call ends in the call stack.
Your example looks like C; in C, there is always a main function; even the main function ends on the call stack.
I don't think there is a way to examine the call stack in C; especially since the compiler is allowed to optimise away whatever it wants. For instance, it could optimise tail-recursion, and then the call stack would be smaller than you'd expect.
In Python the call stack is easy to examine; just crash the function whenever you want, by throwing an exception (for instance with assert(False)). Then the program will produce an error message containing the full "stack trace", including the list of every function on the stack.
Here is an example of a stack trace in python:
def fact1(n):
assert(n != 1)
return n * fact1(n-1)
def main():
f = fact1(3)
print(f)
main()
Output:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 2, in main
File "<stdin>", line 3, in fact1
File "<stdin>", line 3, in fact1
File "<stdin>", line 2, in fact1
AssertionError
And another example just for fun:
def print_even(n):
if (n <= 1):
print('yes' if n == 0 else 'no')
assert(False)
else:
print_odd(n-1)
def print_odd(n):
if (n <= 1):
print('yes' if n == 1 else 'no')
assert(False)
else:
print_even(n-1)
def main():
n = 5
print('Is {} even?'.format(n))
print_even(n)
main()
Output:
Is 5 even?
no
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 4, in main
File "<stdin>", line 6, in print_even
File "<stdin>", line 6, in print_odd
File "<stdin>", line 6, in print_even
File "<stdin>", line 6, in print_odd
File "<stdin>", line 4, in print_even
AssertionError

Using writeOGR with rpy2

I am trying to use writeOGR using R called from Python 3.8.
import rpy2.robjects as ro
.....
ro.r('ttops <- .....')
ro.r('writeOGR(obj=ttops, dsn="T:/Internal/segmentation", layer="test", driver="ESRI Shapefile")')
errors with:
R[write to console]: Error in writeOGR(obj = ttops, dsn = "T:/Internal/LiDAR/crown_segmentation", :
could not find function "writeOGR"
Traceback (most recent call last):
File "C:/Users/david/PycharmProjects/main.py", line 7, in <module>
main()
File "C:/Users/david/PycharmProjects/main.py", line 4, in main
R_Packages().process()
File "C:\Users\david\PycharmProjects\model_testing\r_methods.py", line 17, in process
ro.r('writeOGR(obj=ttops, dsn="T:/Internal/segmentation", layer="test", driver="ESRI Shapefile")')
File "C:\Users\david\AppData\Local\Programs\Python\Python38\lib\site-packages\rpy2\robjects\__init__.py", line 416, in __call__
res = self.eval(p)
File "C:\Users\david\AppData\Local\Programs\Python\Python38\lib\site-packages\rpy2\robjects\functions.py", line 197, in __call__
return (super(SignatureTranslatedFunction, self)
File "C:\Users\david\AppData\Local\Programs\Python\Python38\lib\site-packages\rpy2\robjects\functions.py", line 125, in __call__
res = super(Function, self).__call__(*new_args, **new_kwargs)
File "C:\Users\david\AppData\Local\Programs\Python\Python38\lib\site-packages\rpy2\rinterface_lib\conversion.py", line 44, in _
cdata = function(*args, **kwargs)
File "C:\Users\david\AppData\Local\Programs\Python\Python38\lib\site-packages\rpy2\rinterface.py", line 624, in __call__
raise embedded.RRuntimeError(_rinterface._geterrmessage())
rpy2.rinterface_lib.embedded.RRuntimeError: Error in writeOGR(obj = ttops, dsn = "T:/Internal/segmentation", :
could not find function "writeOGR"
Am I missing something or is this a limit of rpy2? If it is a limit, what is an alternative to write shapefiles of R data using Python?
There was a library that I did not need to define in R that was needed in Python:
ro.r('library(rgdal)')

Tuple index out of range Tkinter

So I've got a program that should take a function as input and graph it on a Tkinter canvas.
def draw(self):
self.canvas.delete(ALL)
for n, i in enumerate(self.sav):
self.function, colour = self.sav_func[n]
i = self.p1(i)
i = self.p2(i, self.function, colour)
if i != [0]:
try:
self.canvas.create_line(i, fill = colour)
except TclError as err:
tkMessageBox.showerror(TclError, err)
self.sav.remove(self.sav[len(self.sav)-1])
self.sav_func.remove(self.sav_func[len(self.sav_func)-1])
This section is giving me the following error:
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Python27\lib\lib-tk\Tkinter.py", line 1410, in __call__
return self.func(*args)
File "D:/Google Drive/assign2_2-1.py", line 113, in add_func
self.redraw_all()
File "D:/Google Drive/assign2_2-1.py", line 132, in redraw_all
self.draw()
File "D:/Google Drive/assign2_2-1.py", line 145, in draw
self.canvas.create_line(i, fill = colour)
File "C:\Python27\lib\lib-tk\Tkinter.py", line 2201, in create_line
return self._create('line', args, kw)
File "C:\Python27\lib\lib-tk\Tkinter.py", line 2182, in _create
cnf = args[-1]
IndexError: tuple index out of range
From what I can gather it's something to do with the number of inputs not matching the number of outputs, but I'm still a little lost. Help would be great!
it looks like i doesn't have enough values. To create a line it needs four values: x1,y1,x2,y2.

Resources