So, I created the following HTTP server tunneled via ngrok, and I am trying to send a file to the server, to then read it and display it on the web page of the server.
Here's the code for the server:
import os
from http.server import HTTPServer, BaseHTTPRequestHandler
from pyngrok import ngrok
import time
port = os.environ.get("PORT", 80)
server_address = ("127.0.0.1", port)
class MyServer(BaseHTTPRequestHandler):
def _set_headers(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
def do_GET(self):
self._set_headers()
self.wfile.write(bytes("<html><head><title>https://pythonbasics.org</title></head>", "utf-8"))
self.wfile.write(bytes("<p>Request: %s</p>" % self.path, "utf-8"))
self.wfile.write(bytes("<body>", "utf-8"))
self.wfile.write(bytes("<p>This is an example web server.</p>", "utf-8"))
self.wfile.write(bytes("</body></html>", "utf-8"))
def do_POST(self):
'''Reads post request body'''
self._set_headers()
content_len = int(self.headers.getheader('content-length', 0))
post_body = self.rfile.read(content_len)
self.wfile.write("received post request:<br>{}".format(post_body))
def do_PUT(self):
self.do_POST()
httpd = HTTPServer(server_address, MyServer)
public_url = ngrok.connect(port).public_url
print("ngrok tunnel \"{}\" -> \"http://127.0.0.1:{}\"".format(public_url, port))
try:
# Block until CTRL-C or some other terminating event
httpd.serve_forever()
except KeyboardInterrupt:
print(" Shutting down server.")
httpd.socket.close()
And I have been trying to send a file using POST as follow
>>> url = 'https://httpbin.org/post'
>>> files = {'file': open('report.xls', 'rb')}
>>> r = requests.post(url, files=files)
>>> r.text
I imported requests of course, and here's what I get
Exception occurred during processing of request from ('127.0.0.1', 60603)
Traceback (most recent call last):
File "C:\Program Files\Python39\lib\socketserver.py", line 316, in _handle_request_noblock
self.process_request(request, client_address)
File "C:\Program Files\Python39\lib\socketserver.py", line 347, in process_request
self.finish_request(request, client_address)
File "C:\Program Files\Python39\lib\socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "C:\Program Files\Python39\lib\socketserver.py", line 720, in __init__
self.handle()
File "C:\Program Files\Python39\lib\http\server.py", line 427, in handle
self.handle_one_request()
File "C:\Program Files\Python39\lib\http\server.py", line 415, in handle_one_request
method()
File "C:\Users\pierr\OneDrive\Desktop\SpyWare-20210104T124335Z-001\SpyWare\Ngrok_Test.py", line 28, in do_POST
content_len = int(self.headers.getheader('content-length', 0))
AttributeError: 'HTTPMessage' object has no attribute 'getheader'
Could someone please help me fix this error ? I don't get where it comes from.
Syntax has changed. You need to use
content_len = int(self.headers.get('Content-Length'))
Instead of
content_len = int(self.headers.getheader('content-length', 0))
The rest should be the same
Related
Forgive my ignorance, I'm just starting to use the Python requests module, with version 2.22.0. When I pass a request with verify=False, I get an exception. Is there something I need to set the right knob in urllib3? Is this a case where I need to use a transport adapter? Thanks!
Request:
response = requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=('user','pass'))
Exception:
File "/usr/local/lib/python3.6/site-packages/requests/api.py", line 116, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 376, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 994, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.6/site-packages/urllib3/connection.py", line 374, in connect
context.verify_mode = resolve_cert_reqs(self.cert_reqs)
File "/usr/lib64/python3.6/ssl.py", line 443, in verify_mode
super(SSLContext, SSLContext).verify_mode.__set__(self, value)
ValueError: Cannot set verify_mode to CERT_NONE when check_hostname is enabled.
You can do this to create a session :
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.ssl_ import ssl
class SSLContextAdapter(HTTPAdapter):
def init_poolmanager(self, *args, **kwargs):
context = ssl.create_default_context()
kwargs['ssl_context'] = context
context.load_default_certs() # this loads the OS defaults on Windows
return super(SSLContextAdapter, self).init_poolmanager(*args, **kwargs)
s = requests.Session()
adapter = SSLContextAdapter()
s.mount('yoururl', adapter)
response = s.post('yoururl', data=json.dumps(payload), headers=headers, auth=('user','pass'))
I am trying to download a csv file from the internet. Here is my code using urllib. But I get HTTP Error 403.
Program-1:
from urllib import request
nse_stocks = 'https://www.nseindia.com/products/content/sec_bhavdata_full.csv'
def download_file(url):
connection = request.urlopen(url)
file_read = connection.read()
file_str = str(file_read)
lines_file_str = file_str.split('\\n')
file = open(r'downloaded_file.csv', 'w')
for line in lines_file_str:
file.write(line + '\n')
file.close()
download_file(nse_stocks)
Response:
Traceback (most recent call last):
File "C:/Users/sg0205481/Documents/Krishna/eBooks/Python/TheNewBoston/Python/downloadWebFile2.py", line 17, in <module>
download_file(nse_stocks)
File "C:/Users/sg0205481/Documents/Krishna/eBooks/Python/TheNewBoston/Python/downloadWebFile2.py", line 7, in download_file
connection = request.urlopen(url)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\sg0205481\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
But I don't get the error with Program-2 with requests module. File gets downloaded successfully.
Program-2:
import requests
def download_file(url):
file_data = requests.get(url)
filename = 'downloaded_file.csv'
with open(filename, 'wb') as file:
file.write(file_data.content)
download_file('https://www.nseindia.com/products/content/sec_bhavdata_full.csv')
What is the problem with Program-1? What makes Program-2 pass successfully?
You can download a file (mime type) with get method from web server, not post method. If web server has handler for this extension and accept post method, than you may try!!!
My code below works fine with Linux servers but the script doesn't work with Windows 2008 Server with OpenSSH installed. I have also tested to ssh with teraterm with the same username, password, ipaddress and port22 it worked fine as well.
I am getting the below error.
C:\Users\>python auto-ssh_v1.py
Socket exception: An existing connection was forcibly closed by the remote host (10054)
Traceback (most recent call last):
File "auto-ssh_v1.py", line 71, in <module>
results = executer.execute()
File "auto-ssh_v1.py", line 53, in execute
stdin, stdout, stderr = ssh.exec_command(self.command)
File "C:\Anaconda3\lib\site-packages\paramiko\client.py", line 405, in exec_command
chan.exec_command(command)
File "C:\Anaconda3\lib\site-packages\paramiko\channel.py", line 60, in _check
return func(self, *args, **kwds)
File "C:\Anaconda3\lib\site-packages\paramiko\channel.py", line 229, in exec_command
self._wait_for_event()
File "C:\Anaconda3\lib\site-packages\paramiko\channel.py", line 1086, in _wait_for_event
raise e
File "C:\Anaconda3\lib\site-packages\paramiko\transport.py", line 1726, in run
ptype, m = self.packetizer.read_message()
File "C:\U\Anaconda3\lib\site-packages\paramiko\packet.py", line 386, in read_message
header = self.read_all(self.__block_size_in, check_rekey=True)
File "C:\Anaconda3\lib\site-packages\paramiko\packet.py", line 249, in read_all
x = self.__socket.recv(n)
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
The code is
#Modules
import paramiko
#Variables
USER = 'Administrator'
PSWD = 'Passw0rd'
#Classes and Functions
class InputReader:
def __init__(self, commands_path, hosts_path):
self.commands_path = commands_path
self.hosts_path = hosts_path
def read(self):
self.commands = self.__readlines(self.commands_path)
self.hosts = self.__readlines(self.hosts_path)
def __readlines(self, path):
with open(path) as f:
return [v.strip() for v in f.readlines()] #List comprehension
class CommandExecuter:
def __init__(self, host, command):
self.host = host
self.command = command
def execute(self):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(self.host, username=USER, password=PSWD)
stdin, stdout, stderr = ssh.exec_command(self.command)
errors = stderr.readlines()
if len(errors) != 0:
raise Exception(errors)
lines = [v.strip() for v in stdout.readlines()]
ssh.close()
return lines
#Main Procedure
if __name__ == '__main__':
reader = InputReader("commands.txt", "systems.txt")
reader.read()
for h in reader.hosts:
for c in reader.commands:
executer = CommandExecuter(h, c)
results = executer.execute()
print("{0}({1}):".format(h, c))
for i in results:
print(i)
print('\n')
You can try to run paramiko line by line to ssh to your windows server and see the trace.
Hope that will help,
Trinh
urllib3 V1.8.3 has been added the feature "socket_options". For me, I want to specify (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) in requests lib. But I've no idea how.
Here is the version change log of urllib3.
Here is the sample of the new feature in urllib3:
def test_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=[
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
])
s = pool._new_conn()._new_conn() # Get the socket
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(using_keepalive)
s.close()
If version 1.8.3 is merged into new requests lib, how to specify the parameter.
Any answers will be appreciated.
Here is my code:
import requests
import socket
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.poolmanager import PoolManager
class SourceAddressAdapter(HTTPAdapter):
def __init__(self, source_address, **kwargs):
self.source_address = source_address
super(SourceAddressAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)],
source_address=self.source_address,
)
http_session = requests.Session()
http_session.mount('http://', SourceAddressAdapter(('0.0.0.0', 1238)))
print http_session.get('http://10.0.10.7')
And here is the error:
Traceback (most recent call last):
File "./tt.py", line 27, in <module>
print http_session.get('http://10.0.10.7')
File "/usr/local/lib/python2.7/site-packages/requests-2.3.0-py2.7.egg/requests/sessions.py", line 468, in get
return self.request('GET', url, **kwargs)
File "/usr/local/lib/python2.7/site-packages/requests-2.3.0-py2.7.egg/requests/sessions.py", line 456, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python2.7/site-packages/requests-2.3.0-py2.7.egg/requests/sessions.py", line 559, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python2.7/site-packages/requests-2.3.0-py2.7.egg/requests/adapters.py", line 375, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='10.0.10.7', port=80): Max retries exceeded with url: / (Caused by <class 'socket.error'>: [Errno 98] Address already in use)
===============================================================================
I have tried to use urllib3 v1.9 to set the (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1), but it failed. The reason why source port in time_wait state can't be reused may be "(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)" should be configured before "socket.bind". So even I set the REUSEADDR parameter in the "HTTPConnectionPool", src port in time_wait state could NOT be used. So sad..
Here is my try:
#!/usr/bin/python
import socket
from urllib3 import (
encode_multipart_formdata,
HTTPConnectionPool,
)
pool = HTTPConnectionPool('10.0.10.7', 80,
socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)],
source_address=('10.0.10.11',1239))
r=pool.request('GET','/')
And here is the error:
Traceback (most recent call last):
File "./aa.py", line 10, in <module>
r=pool.request('GET','/')
File "build/bdist.linux-x86_64/egg/urllib3/request.py", line 68, in request
File "build/bdist.linux-x86_64/egg/urllib3/request.py", line 81, in request_encode_url
File "build/bdist.linux-x86_64/egg/urllib3/connectionpool.py", line 579, in urlopen
File "build/bdist.linux-x86_64/egg/urllib3/connectionpool.py", line 579, in urlopen
File "build/bdist.linux-x86_64/egg/urllib3/connectionpool.py", line 579, in urlopen
File "build/bdist.linux-x86_64/egg/urllib3/connectionpool.py", line 559, in urlopen
File "build/bdist.linux-x86_64/egg/urllib3/util/retry.py", line 265, in increment
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='10.0.10.7', port=80): Max retries exceeded with url: / (Caused by ProtocolError('Connection aborted.', error(98, 'Address already in use')))
I couldn't get #Lukasa's answer to work, but I modified it and got the following to work,
import requests
import socket
from requests.adapters import HTTPAdapter
from requests.adapters import PoolManager
from requests.packages.urllib3.connection import HTTPConnection
class SockOpsAdapter(HTTPAdapter):
def __init__(self, options, **kwargs):
self.options = options
super(SockOpsAdapter, self).__init__()
def init_poolmanager(self, connections, maxsize, block=False):
print "init_poolmanager"
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
socket_options=self.options)
options = HTTPConnection.default_socket_options + [
(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
]
print "build session"
s = requests.Session()
s.mount('http://', SockOpsAdapter(options))
s.mount('https://', SockOpsAdapter(options))
for i in xrange(0, 10):
print "sending request %i" % i
url = 'http://host:port' #put in a host/port here
headers = {'Content-Type':'text/plain', 'Accept':'text/plain'}
post_status = s.get(url, headers=headers)
print "Post Status Code = %s" % str(post_status.status_code)
print post_status.content[0:50]
Assuming you've correctly merged urllib3 into requests and that nothing breaks, you'd use a Transport Adapter:
from requests.adapters import HTTPAdapter
class SockOpsAdapter(HTTPAdapter):
def __init__(self, options, **kwargs):
self.options = options
super(SourceAddressAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
socket_options=self.options)
And then you use it like this:
import requests
from requests.packages.urllib3.connection import HTTPConnection
options = HTTPConnection.default_socket_options + [
(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1),
]
s = requests.Session()
s.mount('http://', SockOpsAdapter(options))
s.mount('https://', SockOpsAdapter(options))
I am attempting to use Python3 to send metrics to Hosted Graphite. The examples given on the site are Python2, and I have successfully ported the TCP and UDP examples to Python3 (despite my inexperience, and have submitted the examples so the docs may be updated), however I have been unable to get the HTTP method to work.
The Python2 example looks like this:
import urllib2, base64
url = "https://hostedgraphite.com/api/v1/sink"
api_key = "YOUR-API-KEY"
request = urllib2.Request(url, "foo 1.2")
request.add_header("Authorization", "Basic %s" % base64.encodestring(api_key).strip())
result = urllib2.urlopen(request)
This works successfully, returning a HTTP 200.
So far I have ported this much to Python3, and while I was (finally) able to get it to make a valid HTTP request (i.e. no syntax errors), the request fails, returning HTTP 400
import urllib.request, base64
url = "https://hostedgraphite.com/api/v1/sink"
api_key = b'YOUR-API-KEY'
metric = "testing.python3.http 1".encode('utf-8')
request = urllib.request.Request(url, metric)
request.add_header("Authorization", "Basic %s" % base64.encodestring(api_key).strip())
result = urllib.request.urlopen(request)
The full result is:
>>> result = urllib.request.urlopen(request)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 160, in urlopen
return opener.open(url, data, timeout)
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 479, in open
response = meth(req, response)
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 591, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 517, in error
return self._call_chain(*args)
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 451, in _call_chain
result = func(*args)
File "/usr/local/Cellar/python3/3.3.1/Frameworks/Python.framework/Versions/3.3/lib/python3.3/urllib/request.py", line 599, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request
Is it obvious what I am doing wrong? Are there any suggestions on how I might capture and compare what the successful (python2) and failing (python3) requests are actually sending?
Don't mix Unicode strings and bytes:
>>> "abc %s" % b"def"
"abc b'def'"
You could construct the header as follows:
from base64 import b64encode
headers = {'Authorization': b'Basic ' + b64encode(api_key)}
A quick way to see the request is to change the host in the url to localhost:8888 and run before making the request:
$ nc -l 8888
You could also use wireshark to see the requests.