ssh paramiko can't read xlsx files - sftp

I use paramiko to ssh
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy( paramiko.AutoAddPolicy() )
ssh.connect(Hostname, username=Username, password=Password)
ftp = ssh.open_sftp()
files = ftp.listdir()
dir_oi = "directory_of_interest"
foi = ftp.listdir(dir_oi)
and can find a read a csv successfully with:
remote_file = ( dir_oi +"/" + foi[-1])
with ftp.open(remote_file) as f:
df = pd.read_csv(f, sep = '\t', header = None)
the minute I change remote_file to an xlsx, with
with ftp.open(uw_remote_file) as f:
df = pd.read_excel(f)
I get the error SSHException: Server connection dropped: or Socket Closed
of note, I can run this line without any error existing_xlsx = ftp.open(uw_remote_file)
Any suggestions how to overcome this?
logfile as requested:
DEB [20220519-09:22:45.998] thr=1 paramiko.transport.sftp: [chan 0] listdir(b'blah')
DEB [20220519-09:22:48.009] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah/halb.csv', 'r')
DEB [20220519-09:22:48.241] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah/halb.csv', 'r') -> 35323935333939313533313032363062
DEB [20220519-09:22:49.084] thr=1 paramiko.transport.sftp: [chan 0] close(35323935333939313533313032363062)
DEB [20220519-09:23:24.790] thr=1 paramiko.transport.sftp: [chan 0] listdir(b'blah2')
DEB [20220519-09:24:01.590] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah2/halb2.xlsx', 'r')
DEB [20220519-09:24:01.975] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah2/halb2.xlsx', 'r') -> 37343338363564356234303033663337
DEB [20220519-09:24:23.510] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah2/halb2.xlsx', 'r')
DEB [20220519-09:24:23.727] thr=1 paramiko.transport.sftp: [chan 0] open(b'blah2/halb2.xlsx', 'r') -> 64646361316532373233663463613036
DEB [20220519-09:24:24.108] thr=2 paramiko.transport: EOF in transport thread
DEB [20220519-09:24:24.108] thr=1 paramiko.transport.sftp: [chan 0] close(64646361316532373233663463613036)
traceback:
Traceback (most recent call last):
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp_client.py", line 852, in _read_response
t, data = self._read_packet()
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp.py", line 201, in _read_packet
x = self._read_all(4)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp.py", line 188, in _read_all
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\alexander.huhn.adm\AppData\Local\Temp\24\ipykernel_33560\4051829457.py", line 4, in <cell line: 2>
df = pd.read_excel(f)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\pandas\util\_decorators.py", line 311, in wrapper
return func(*args, **kwargs)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\pandas\io\excel\_base.py", line 457, in read_excel
io = ExcelFile(io, storage_options=storage_options, engine=engine)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\pandas\io\excel\_base.py", line 1376, in __init__
ext = inspect_excel_format(
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\pandas\io\excel\_base.py", line 1255, in inspect_excel_format
buf = stream.read(PEEK_SIZE)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\file.py", line 219, in read
new_data = self._read(read_size)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp_file.py", line 185, in _read
t, msg = self.sftp._request(
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp_client.py", line 822, in _request
return self._read_response(num)
File "C:\Users\alexander.huhn.adm\Anaconda3\lib\site-packages\paramiko\sftp_client.py", line 854, in _read_response
raise SSHException("Server connection dropped: {}".format(e))
paramiko.ssh_exception.SSHException: Server connection dropped:
I can download using
filepath = uw_remote_file
localpath = "test.xlsx"
ftp.get(filepath,localpath)
so will go down that route and delete after use

From the log it looks like it were able to open both files. pandas is running into EOF error, So, which means excel file is completely empty.
Can you confirm if that is empty.

Related

Gremlin/Python: run query as string

I have the following code, and run as expected. But I need to use the "g" traversal object to manipulate the graph.
from gremlin_python.process.anonymous_traversal import traversal
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
g = traversal().withRemote(DriverRemoteConnection('ws://localhost:8182/gremlin','g'))
g.V().drop().iterate()
g.addV('my-label').property('k', 'v').next()
print(g.V().toList())
Instead of the "g" object, I want to run string query to modify the graph, and the following doesn't work.
from gremlin_python.driver import client
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
ws_conn = DriverRemoteConnection('ws://localhost:8182/gremlin','g')
gremlin_conn = client.Client(ws_conn, "g")
query = "g.V().groupCount().by(label).unfold().project('label','count').by(keys).by(values)"
response = gremlin_conn.submit(query)
print(response)
Gives the following error:
(venv) sh-3.2$ python /Users/demo-prj/tests/tools/neptune/local.py
[v[4280]]
Traceback (most recent call last):
File "/Users/demo-prj/tests/tools/neptune/local.py", line 24, in <module>
response = gremlin_conn.submit(query)
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/client.py", line 127, in submit
return self.submitAsync(message, bindings=bindings, request_options=request_options).result()
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/client.py", line 148, in submitAsync
return conn.write(message)
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/connection.py", line 55, in write
self.connect()
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/connection.py", line 45, in connect
self._transport.connect(self._url, self._headers)
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/tornado/transport.py", line 40, in connect
self._ws = self._loop.run_sync(
File "/Users/demo-prj/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 576, in run_sync
return future_cell[0].result()
File "/Users/demo-prj/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 547, in run
result = func()
File "/Users/demo-prj/venv/lib/python3.8/site-packages/gremlin_python/driver/tornado/transport.py", line 41, in <lambda>
lambda: websocket.websocket_connect(url, compression_options=self._compression_options))
File "/Users/demo-prj/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1333, in websocket_connect
conn = WebSocketClientConnection(request,
File "/Users/demo-prj/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1122, in __init__
scheme, sep, rest = request.url.partition(':')
AttributeError: 'DriverRemoteConnection' object has no attribute 'partition'
This works.
from gremlin_python.driver import client
from tornado import httpclient
ws_url = 'ws://localhost:8182/gremlin'
ws_conn = httpclient.HTTPRequest(ws_url)
gremlin_conn = client.Client(ws_conn, "g")
query = "g.V().groupCount().by(label).unfold().project('label','count').by(keys).by(values)"
response = gremlin_conn.submit(query)
print(response)

Jupyter R Kernel Failing to Start

After installing the R kernel via conda, I get the following error when trying to start up a Kernel. The error says there's a file missing, but I can't figure out what file it's referring to. Any idea what's actually throwing the error?
Traceback (most recent call last):
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/web.py", line 1445, in _execute
result = yield result
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/concurrent.py", line 232, in result
raise_exc_info(self._exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1014, in run
yielded = self.gen.throw(*exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/notebook/services/sessions/handlers.py", line 73, in post
type=mtype))
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/concurrent.py", line 232, in result
raise_exc_info(self._exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1014, in run
yielded = self.gen.throw(*exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/notebook/services/sessions/sessionmanager.py", line 79, in create_session
kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/concurrent.py", line 232, in result
raise_exc_info(self._exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1014, in run
yielded = self.gen.throw(*exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/notebook/services/sessions/sessionmanager.py", line 92, in start_kernel_for_session
self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 1008, in run
value = future.result()
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/concurrent.py", line 232, in result
raise_exc_info(self._exc_info)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/tornado/gen.py", line 282, in wrapper
yielded = next(result)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/notebook/services/kernels/kernelmanager.py", line 141, in start_kernel
super(MappingKernelManager, self).start_kernel(**kwargs)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/jupyter_client/multikernelmanager.py", line 109, in start_kernel
km.start_kernel(**kwargs)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/jupyter_client/manager.py", line 244, in start_kernel
**kw)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/jupyter_client/manager.py", line 190, in _launch_kernel
return launch_kernel(kernel_cmd, **kw)
File "/home/Jupyter/anaconda2/lib/python2.7/site-packages/jupyter_client/launcher.py", line 123, in launch_kernel
proc = Popen(cmd, **kwargs)
File "/home/Jupyter/anaconda2/lib/python2.7/subprocess.py", line 390, in __init__
errread, errwrite)
File "/home/Jupyter/anaconda2/lib/python2.7/subprocess.py", line 1025, in _execute_child
raise child_exception
OSError: [Errno 2] No such file or directory

"Failed to import pydot" throws in kerasR

I use the package keras under R and I would like to know if there was a command like python with plot_model () which allows to display its neuron network
library(keras)
for example I would like to display this neural network under R
model <- keras_model_sequential()
model %>%
layer_dense(units = 5, input_shape = 2) %>%
layer_activation("relu") %>%
layer_dense(units = 1)
I install package kerasR for use the function plot_model(), but i have this error.
> library(kerasR)
> plot_model(model)
Error in py_call_impl(callable, dots$args, dots$keywords) :
ImportError: Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.
Detailed traceback:
File "C:\Users\Idriss\ANACON~1\envs\R-TENS~1\lib\site-packages\keras\utils\vis_utils.py", line 131, in plot_model
dot = model_to_dot(model, show_shapes, show_layer_names, rankdir)
File "C:\Users\Idriss\ANACON~1\envs\R-TENS~1\lib\site-packages\keras\utils\vis_utils.py", line 52, in model_to_dot
_check_pydot()
File "C:\Users\Idriss\ANACON~1\envs\R-TENS~1\lib\site-packages\keras\utils\vis_utils.py", line 27, in _check_pydot
raise ImportError('Failed to import pydot. You must install pydot'
I'm use windows 10 64 bits, i use RStudio with Anaconda
In [4] pydot.Dot.create(pydot.Dot())
Out[4]: b"%!PS-Adobe-3.0\r\n%%Creator: graphviz version 2.38.0 (20140413.2041)\r\n%%Title: G\r\n%%Pages: (atend)\r\n%%BoundingBox: (atend)\r\n%%EndComments\r\nsave\r\n%%BeginProlog\r\n/DotDict 200 dict def\r\nDotDict begin\r\n\r\n/setupLatin1 {\r\nmark\r\n/EncodingVector 256 array def\r\n EncodingVector 0\r\n\r\nISOLatin1Encoding 0 255 getinterval putinterval\r\nEncodingVector 45 /hyphen put\r\n\r\n% Set up ISO Latin 1 character encoding\r\n/starnetISO {\r\n dup dup findfont dup length dict begin\r\n { 1 index /FID ne { def }{ pop pop } ifelse\r\n } forall\r\n /Encoding EncodingVector def\r\n currentdict end definefont\r\n} def\r\n/Times-Roman starnetISO def\r\n/Times-Italic starnetISO def\r\n/Times-Bold starnetISO def\r\n/Times-BoldItalic starnetISO def\r\n/Helvetica starnetISO def\r\n/Helvetica-Oblique starnetISO def\r\n/Helvetica-Bold starnetISO def\r\n/Helvetica-BoldOblique starnetISO def\r\n/Courier starnetISO def\r\n/Courier-Oblique starnetISO def\r\n/Courier-Bold starnetISO def\r\n/Courier-BoldOblique starnetISO def\r\ncleartomark\r\n} bind def\r\n\r\n%%BeginResource: procset graphviz 0 0\r\n/coord-font-family /Times-Roman def\r\n/default-font-family /Times-Roman def\r\n/coordfont coord-font-family findfont 8 scalefont def\r\n\r\n/InvScaleFactor 1.0 def\r\n/set_scale {\r\n dup 1 exch div /InvScaleFactor exch def\r\n scale\r\n} bind def\r\n\r\n% styles\r\n/solid { [] 0 setdash } bind def\r\n/dashed { [9 InvScaleFactor mul dup ] 0 setdash } bind def\r\n/dotted { [1 InvScaleFactor mul 6 InvScaleFactor mul] 0 setdash } bind def\r\n/invis {/fill {newpath} def /stroke {newpath} def /show {pop newpath} def} bind def\r\n/bold { 2 setlinewidth } bind def\r\n/filled { } bind def\r\n/unfilled { } bind def\r\n/rounded { } bind def\r\n/diagonals { } bind def\r\n/tapered { } bind def\r\n\r\n% hooks for setting color \r\n/nodecolor { sethsbcolor } bind def\r\n/edgecolor { sethsbcolor } bind def\r\n/graphcolor { sethsbcolor } bind def\r\n/nopcolor {pop pop pop} bind def\r\n\r\n/beginpage {\t% i j npages\r\n\t/npages exch def\r\n\t/j exch def\r\n\t/i exch def\r\n\t/str 10 string def\r\n\tnpages 1 gt {\r\n\t\tgsave\r\n\t\t\tcoordfont setfont\r\n\t\t\t0 0 moveto\r\n\t\t\t(\\() show i str cvs show (,) show j str cvs show (\\)) show\r\n\t\tgrestore\r\n\t} if\r\n} bind def\r\n\r\n/set_font {\r\n\tfindfont exch\r\n\tscalefont setfont\r\n} def\r\n\r\n% draw text fitted to its expected width\r\n/alignedtext {\t\t\t% width text\r\n\t/text exch def\r\n\t/width exch def\r\n\tgsave\r\n\t\twidth 0 gt {\r\n\t\t\t[] 0 setdash\r\n\t\t\ttext stringwidth pop width exch sub text length div 0 text ashow\r\n\t\t} if\r\n\tgrestore\r\n} def\r\n\r\n/boxprim {\t\t\t\t% xcorner ycorner xsize ysize\r\n\t\t4 2 roll\r\n\t\tmoveto\r\n\t\t2 copy\r\n\t\texch 0 rlineto\r\n\t\t0 exch rlineto\r\n\t\tpop neg 0 rlineto\r\n\t\tclosepath\r\n} bind def\r\n\r\n/ellipse_path {\r\n\t/ry exch def\r\n\t/rx exch def\r\n\t/y exch def\r\n\t/x exch def\r\n\tmatrix currentmatrix\r\n\tnewpath\r\n\tx y translate\r\n\trx ry scale\r\n\t0 0 1 0 360 arc\r\n\tsetmatrix\r\n} bind def\r\n\r\n/endpage { showpage } bind def\r\n/showpage { } def\r\n\r\n/layercolorseq\r\n\t[\t% layer color sequence - darkest to lightest\r\n\t\t[0 0 0]\r\n\t\t[.2 .8 .8]\r\n\t\t[.4 .8 .8]\r\n\t\t[.6 .8 .8]\r\n\t\t[.8 .8 .8]\r\n\t]\r\ndef\r\n\r\n/layerlen layercolorseq length def\r\n\r\n/setlayer {/maxlayer exch def /curlayer exch def\r\n\tlayercolorseq curlayer 1 sub layerlen mod get\r\n\taload pop sethsbcolor\r\n\t/nodecolor {nopcolor} def\r\n\t/edgecolor {nopcolor} def\r\n\t/graphcolor {nopcolor} def\r\n} bind def\r\n\r\n/onlayer { curlayer ne {invis} if } def\r\n\r\n/onlayers {\r\n\t/myupper exch def\r\n\t/mylower exch def\r\n\tcurlayer mylower lt\r\n\tcurlayer myupper gt\r\n\tor\r\n\t{invis} if\r\n} def\r\n\r\n/curlayer 0 def\r\n\r\n%%EndResource\r\n%%EndProlog\r\n%%BeginSetup\r\n14 default-font-family set_font\r\n1 setmiterlimit\r\n% /arrowlength 10 def\r\n% /arrowwidth 5 def\r\n\r\n% make sure pdfmark is harmless for PS-interpreters other than Distiller\r\n/pdfmark where {pop} {userdict /pdfmark /cleartomark load put} ifelse\r\n% make '<<' and '>>' safe on PS Level 1 devices\r\n/languagelevel where {pop languagelevel}{1} ifelse\r\n2 lt {\r\n userdict (<<) cvn ([) cvn load put\r\n userdict (>>) cvn ([) cvn load put\r\n} if\r\n\r\n%%EndSetup\r\nsetupLatin1\r\n%%Page: 1 1\r\n%%PageBoundingBox: 36 36 44 44\r\n%%PageOrientation: Portrait\r\n0 0 1 beginpage\r\ngsave\r\n36 36 8 8 boxprim clip newpath\r\n1 1 set_scale 0 rotate 40 40 translate\r\nendpage\r\nshowpage\r\ngrestore\r\n%%PageTrailer\r\n%%EndPage: 1\r\n%%Trailer\r\n%%Pages: 1\r\n%%BoundingBox: 36 36 44 44\r\nend\r\nrestore\r\n%%EOF\r\n"
First thing first, the error:
File "C:\Users\Idriss\ANACON~1\envs\R-TENS~1\lib\site-packages\keras\utils\vis_utils.py", line 52, in model_to_dot
_check_pydot()
If we check the file C:\Users\Idriss\ANACON~1\envs\R-TENS~1\lib\site-packages\keras\utils\vis_utils.py and search for the function _check_pydot():
def _check_pydot():
try:
# Attempt to create an image of a blank graph
# to check the pydot/graphviz installation.
pydot.Dot.create(pydot.Dot())
except Exception:
# pydot raises a generic Exception here,
# so no specific class can be caught.
raise ImportError('Failed to import pydot. You must install pydot'
' and graphviz for `pydotprint` to work.')
This error message is lack of information since it catch ALL exception instead of specific execption and raise hard-coded error ImportError(Failed to import blah blah).
To ensure it import the relevant pydot, we should also check import part in that file (Rerun R and library(kerasR) to test):
import os
print("hole 0")
try:
# pydot-ng is a fork of pydot that is better maintained.
import pydot_ng as pydot
print("hole 1")
except ImportError:
# pydotplus is an improved version of pydot
try:
print("hole 1.2")
import pydotplus as pydot
print("hole 2")
except ImportError:
# Fall back on pydot if necessary.
try:
print("hole 3")
import pydot
except ImportError:
print("hole 4")
pydot = None
print("hole -1: " + str(locals())) #alternative way to debug
...
Tips: The safer way to debug is userepr instead of str.
If you manually run python in interactive mode and do pydot.Dot.create(pydot.Dot()), you will find out the exact exception (below is my Linux sample):
xb#dnxb:~/anaconda3/envs/r-tensorflow/bin$ ./python
Python 3.6.3 |Anaconda, Inc.| (default, Nov 20 2017, 20:41:42)
[GCC 7.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import pydot
>>> pydot.Dot()
<pydot.Dot object at 0x7f7d045cdb38>
>>> pydot.Dot
<class 'pydot.Dot'>
>>> pydot.Dot.create(pydot.Dot())
Traceback (most recent call last):
File "/home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/site-packages/pydot.py", line 1878, in create
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
File "/home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/subprocess.py", line 709, in __init__
restore_signals, start_new_session)
File "/home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/subprocess.py", line 1344, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'dot': 'dot'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/site-packages/pydot.py", line 1883, in create
prog=prog))
Exception: "dot" not found in path.
>>>
Let's print some variables used in the file /home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/site-packages/pydot.py before the line 1878:
try:
print("env: " + str(env))
print("cmdline: " + str(cmdline))
print("tmp_dir: " + str(tmp_dir))
p = subprocess.Popen(
cmdline,
env=env,
cwd=tmp_dir,
shell=False,
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
except OSError as e:
if e.errno == os.errno.ENOENT:
raise Exception(
'"{prog}" not found in path.'.format(
prog=prog))
else:
raise
Restart your python interpreter, rerun the import pydot and pydot.Dot.create(pydot.Dot()), it will shows:
xb#dnxb:~/anaconda3/envs/r-tensorflow/bin$ ./python
Python 3.6.3 |Anaconda, Inc.| (default, Nov 20 2017, 20:41:42)
[GCC 7.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import pydot
>>> pydot.Dot.create(pydot.Dot())
env: {'PATH': '/home/xiaobai/anaconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:...<My other PATH>'}
cmdline: ['dot', '-Tps', '/tmp/tmpffo17gx5']
tmp_dir: /tmp
Traceback (most recent call last):
File "/home/xiaobai/anaconda3/envs/r-tensorflow/lib/python3.6/site-packages/pydot.py", line 1881, in create
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
So basically what it does is run the command dot -Tps /tmp/tmpffo17gx5 but failed due to dot command not found.
In Linux, it will suggest run sudo apt install graphviz to install dot if I run the command manually in terminal:
xb#dnxb:~/anaconda3/envs/r-tensorflow/bin$ dot
The program 'dot' is currently not installed. You can install it by typing:
sudo apt install graphviz
xb#dnxb:~/anaconda3/envs/r-tensorflow/bin$ sudo apt install graphviz
...
Run dot -Tps /tmp/tmpffo17gx5 will success now:
xb#dnxb:~/anaconda3/envs/r-tensorflow/bin$ dot -Tps /tmp/tmpffo17gx5
%!PS-Adobe-3.0
%%Creator: graphviz version 2.38.0 (20140413.2041)
%%Title: G
%%Pages: (atend)
%%BoundingBox: (atend)
%%EndComments
save
%%BeginProlog
/DotDict 200 dict def
DotDict begin
...
Restart R session, no more error:
> plot_model(model)
>
This sudo apt install graphviz is for Linux, but I hope this answer help you debug the error in Windows.
You should install the Python libraries:
pip install pydot graphviz
And also you need to download the graphviz binaries, and these are not installed with Python.
On Ubuntu you can install them with apt:
apt-get install -y graphviz libgraphviz-dev
On osX with brew:
brew install graphviz
For Windows and other operating systems, the instructions can be found at http://www.graphviz.org/

PySpark map datetime to DoW

I'm trying to map a column 'eventtimestamp' to its day of week with the following function:
from datetime import datetime
import calendar
from pyspark.sql.functions import UserDefinedFunction as udf
def toWeekDay(x):
v = int(datetime.strptime(str(x),'%Y-%m-%d %H:%M:%S').strftime('%w'))
if v == 0:
v = 6
else:
v = v-1
return calendar.day_name[v]
and for my df trying to create a new column dow with UDF.
udf_toWeekDay = udf(lambda x: toWeekDay(x), StringType())
df = df.withColumn("dow",udf_toWeekDay('eventtimestamp'))
Yet, I'm getting error I do not understand at all. Firstly, it was complaining for inserting datetime.datetime into strptime instead of string. So I parsed to str and now I don't have a clue what's wrong.
Traceback (most recent call last):
File "/tmp/zeppelin_pyspark-9040214714346906648.py", line 267, in <module>
raise Exception(traceback.format_exc())
Exception: Traceback (most recent call last):
File "/tmp/zeppelin_pyspark-9040214714346906648.py", line 260, in <module>
exec(code)
File "<stdin>", line 10, in <module>
File "/usr/lib/spark/python/pyspark/sql/dataframe.py", line 429, in take
return self.limit(num).collect()
File "/usr/lib/spark/python/pyspark/sql/dataframe.py", line 391, in collect
port = self._jdf.collectToPython()
File "/usr/lib/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py", line 1133, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/usr/lib/spark/python/pyspark/sql/utils.py", line 63, in deco
return f(*a, **kw)
File "/usr/lib/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py", line 319, in get_return_value
format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling o6250.collectToPython.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1107.0 failed 4 times, most recent failure: Lost task 0.3 in stage 1107.0 (TID 63757, ip-172-31-27-113.eu-west-1.compute.internal, executor 819): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
Thanks a lot for clues!
we can use date_format to get dayofweek,
df = df.withColumn("dow",date_format(df['eventtimestamp'],'EEEE'))

Error in sage tutorial coding theory

I have just installed sage 6.3 in Ubuntu 14.04 and I have tried the tutorial in coding theory as follow:
MS = MatrixSpace(GF(2),4,7)
G = MS([[1,1,1,0,0,0,0], [1,0,0,1,1,0,0], [0,1,0,1,0,1,0], [1,1,0,1,0,0,1]])
C = LinearCode(G)
In the third evaluation, sage produced error as follow:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "_sage_input_4.py", line 10, in <module>
exec compile(u'open("___code___.py","w").write("# -*- coding: utf-8 -*-\\n" + _support_.preparse_worksheet_cell(base64.b64decode("QyA9IExpbmVhckNvZGUoRyk="),globals())+"\\n"); execfile(os.path.abspath("___code___.py"))
File "", line 1, in <module>
File "/tmp/tmpXyxNvC/___code___.py", line 2, in <module>
exec compile(u'C = LinearCode(G)
File "", line 1, in <module>
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/coding/linear_code.py", line 785, in __init__
facade_for = gen_mat.row(0).parent()
File "matrix_mod2_dense.pyx", line 576, in sage.matrix.matrix_mod2_dense.Matrix_mod2_dense.row (build/cythonized/sage/matrix/matrix_mod2_dense.c:5387)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 432, in VectorSpace
return FreeModule(K, rank=dimension, sparse=sparse, inner_product_matrix=inner_product_matrix)
File "factory.pyx", line 366, in sage.structure.factory.UniqueFactory.__call__ (build/cythonized/sage/structure/factory.c:1327)
File "factory.pyx", line 410, in sage.structure.factory.UniqueFactory.get_object (build/cythonized/sage/structure/factory.c:1679)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 380, in create_object
return FreeModule_ambient_field(base_ring, rank, sparse=sparse)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 4972, in __init__
FreeModule_ambient_pid.__init__(self, base_field, dimension, sparse=sparse)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 4893, in __init__
FreeModule_ambient_domain.__init__(self, base_ring=base_ring, rank=rank, sparse=sparse)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 4709, in __init__
FreeModule_ambient.__init__(self, base_ring, rank, sparse)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 4184, in __init__
FreeModule_generic.__init__(self, base_ring, rank=rank, degree=rank, sparse=sparse)
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 714, in __init__
self.element_class()
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 896, in element_class
C = element_class(self.base_ring(), self.is_sparse())
File "/usr/local/sage/local/lib/python2.7/site-packages/sage/modules/free_module.py", line 6721, in element_class
import sage.modules.vector_real_double_dense
File "vector_real_double_dense.pyx", line 1, in init sage.modules.vector_real_double_dense (build/cythonized/sage/modules/vector_real_double_dense.c:5611)
File "__init__.pxd", line 155, in init sage.modules.vector_double_dense (build/cythonized/sage/modules/vector_double_dense.c:11813)
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/__init__.py", line 153, in <module>
from . import add_newdocs
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/add_newdocs.py", line 13, in <module>
from numpy.lib import add_newdoc
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/lib/__init__.py", line 18, in <module>
from .polynomial import *
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/lib/polynomial.py", line 19, in <module>
from numpy.linalg import eigvals, lstsq, inv
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/linalg/__init__.py", line 50, in <module>
from .linalg import *
File "/usr/local/sage/local/lib/python2.7/site-packages/numpy/linalg/linalg.py", line 29, in <module>
from numpy.linalg import lapack_lite, _umath_linalg
ImportError: libgfortran.so.3: cannot open shared object file: No such file or directory
How can I solve this problem? I am new to sage!

Resources