I need to run a websocket server on ESP32 and the official example raises the following exception when I connect from any client:
MPY: soft reboot
Network config: ('192.168.0.200', '255.255.255.0', '192.168.0.1', '8.8.8.8')
b'Sec-WebSocket-Version: 13\r\n'
b'Sec-WebSocket-Key: k5Lr79cZgBQg7irI247FMw==\r\n'
b'Connection: Upgrade\r\n'
b'Upgrade: websocket\r\n'
b'Sec-WebSocket-Extensions: permessage-deflate; client_max_window_bits\r\n'
b'Host: 192.168.0.200\r\n'
b'\r\n'
Finished webrepl handshake
Task exception wasn't retrieved
future: <Task> coro= <generator object 'echo' at 3ffe79b0>
Traceback (most recent call last):
File "uasyncio/core.py", line 1, in run_until_complete
File "main.py", line 22, in echo
File "uasyncio/websocket/server.py", line 60, in WSReader
AttributeError: 'Stream' object has no attribute 'ios'
My micropython firmware and libraries:
Micropython firmware: https://micropython.org/resources/firmware/esp32-idf3-20200902-v1.13.bin
Pip libraries installed: micropython-ulogging, uasyncio.websocket.server
My main.py:
import network
import machine
sta_if = network.WLAN(network.STA_IF)
sta_if.active(True)
sta_if.ifconfig(('192.168.0.200', '255.255.255.0', '192.168.0.1', '8.8.8.8'))
if not sta_if.isconnected():
print('connecting to network...')
sta_if.connect('my-ssid', 'my-password')
while not sta_if.isconnected():
machine.idle() # save power while waiting
print('Network config:', sta_if.ifconfig())
# from https://github.com/micropython/micropython-lib/blob/master/uasyncio.websocket.server/example_websock.py
import uasyncio
from uasyncio.websocket.server import WSReader, WSWriter
def echo(reader, writer):
# Consume GET line
yield from reader.readline()
reader = yield from WSReader(reader, writer)
writer = WSWriter(reader, writer)
while 1:
l = yield from reader.read(256)
print(l)
if l == b"\r":
await writer.awrite(b"\r\n")
else:
await writer.awrite(l)
import ulogging as logging
#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
loop = uasyncio.get_event_loop()
loop.create_task(uasyncio.start_server(echo, "0.0.0.0", 80))
loop.run_forever()
loop.close()
MicroPython 1.13 implements asyncio v3 which has a number of breaking changes compared to the 3 year old sample referenced.
I suggest you refer to Peter Hinch's excellent documentation on asyncio,
and the asyncio V3 tutorial
I encountered the same problem. I looked at the old implementation of Stream class [1] and the new one [2].
It seems to me, that you need to edit server.py from uasyncio/websocket/.
You can download the files from [3] to your PC. Then at the bottom of the file replace the two instances of "reader.ios" by "reader.s".
Save the file to your ESP32 and it should work. Of course you need to use "from server import WSReader, WSWriter" instead of "from uasyncio.websocket.server import WSReader, WSWriter".
[1] https://github.com/pfalcon/pycopy-lib/blob/master/uasyncio/uasyncio/__init__.py
[2] https://github.com/micropython/micropython/blob/master/extmod/uasyncio/stream.py
[3] https://pypi.org/project/micropython-uasyncio.websocket.server/#files
https://github.com/pfalcon/pycopy-lib/tree/master/uasyncio has a recent (may'21) sample that should also work on standard MicroPython.
or checkout https://awesome-micropython.com under web servers
Related
I'm Bart and I am new into Python and this is my first post here.
As a fan of whisky I wanted to scrape some shops to give me recent deals on whisky, however, I stuck with Asda's page. I browsed here for ages but without any luck hence my post.
Thank you.
Browser is opening, and closing as expected.
below is my creation:
Import libraries
# import urllib.request
from bs4 import BeautifulSoup
from selenium import webdriver
import time
# import pandas as pd
# import requests
from selenium.webdriver.firefox.options import Options as FirefoxOptions
# specify url
#url = "https://groceries.asda.com/product/whisky/glenmorangie-the-original-single-malt-scotch-whisky/68303869"
url = "https://groceries.asda.com/search/whisky/1/relevance-desc/so-false/Type%3A3612046177%3AMalt%20Whisky"
# run webdriver with headless option
options = FirefoxOptions()
driver = webdriver.Firefox(options=options)
options.add_argument('--headless')
# get page
driver.get(url)
# execute script to scroll down the page
driver.execute_script('window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;')
# sleep for 30s
time.sleep(30)
# close driver
driver.close()
# find element by xpath
results = driver.find_elements_by_xpath("//*[#id='componentsContainer']//*[#id='listingsContainer']//*[#class='product active']//*[#class='title productTitle']")
"""soup = BeautifulSoup(browser.page_source, 'html.parser')"""
print('Number of results', len(results))
Here is the output.
Traceback (most recent call last):
File "D:/PycharmProjects/Giraffe/asda.py", line 29, in <module>
results = driver.find_elements_by_xpath("//*[#id='componentsContainer']//*[#id='listingsContainer']//*[#class='product active']//*[#class='title productTitle']")
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 410, in find_elements_by_xpath
return self.find_elements(by=By.XPATH, value=xpath)
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 1007, in find_elements
'value': value})['value'] or []
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.InvalidSessionIdException: Message: Tried to run command without establishing a connection
Process finished with exit code 1
I tried to stick to the way you have already written. Do not go for hardcoded delay as that is always inconsistent. Try to opt for Explicit Wait. That said this is how you can get the result:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
url = "https://groceries.asda.com/search/whisky"
driver = webdriver.Chrome()
wait = WebDriverWait(driver,10)
driver.get(url)
item = wait.until(EC.presence_of_element_located((By.XPATH, "//h3[#class='co-product-list__title']")))
driver.execute_script("arguments[0].scrollIntoView();",item)
results = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//li[contains(#class,'co-item')]//*[#class='co-product__title']/a")))
print('Number of results:', len(results))
driver.quit()
Output:
Number of results: 61
I have just started using unittest in Python for writing test cases, I have a function that makes the connection with SQL server.
sql_connection.py
def getConnection():
connection = pyodbc.connect("Driver={ODBC Driver 13 for SQL Server};"
"Server="+appConfig['sql_server']['server']+";"
"Database="+appConfig['sql_server']['database']+";"
"UID="+appConfig['sql_server']['uid']+";"
"PWD="+appConfig['sql_server']['password']+";"
"Trusted_Connection=no;",
)
return connection
I have tried below test case for checking database connect or not.
test_connection.py
import pyodbc
getConnection1=getConnection()
class TestDatabseConnection(unittest.TestCase):
def test_getConnection(self):
try:
db_connection = getConnection1.connection
except pyodbc.Error as ex:
sqlstate = ex.args[1]
print(sqlstate)
self.fail(
"getConnection() raised pyodbc.OperationalError. " +
"Connection to database failed. Detailed error message: " + sqlstate)
self.assertIsNone(db_connection)
But still not able to get succeed.
======================================================================
ERROR: test_getConnection (__main__.TestDatabseConnection)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test_connection.py", line 23, in test_getConnection
db_connection = getConnection1.connection
AttributeError: 'pyodbc.Connection' object has no attribute 'connection'
----------------------------------------------------------------------
Ran 1 test in 0.001s
FAILED (errors=1)
Please help me out in this.
A unit test for your getConnection could look like below. I would suggest using a patch and mock from unittest.mock. With a unit test you are only interested in testing the functionality of getConnection and should "Mock" all other function calls within the function. If you want to test the full potential of pyodbc.connect then I would suggest a functional test that actually connects to the databse which would no longer be a unit test. For more information on patch and Mock checkout the docs. These are very powerful and make unit testing fun and easy! unittest.mock
import unittest
from unittest.mock import patch, Mock
import pyodbc
def getConnection():
appConfig = {'sql_server': {'server':'', 'database':'', 'uid':'', 'password':''}}
connection = pyodbc.connect("Driver={ODBC Driver 13 for SQL Server};"
"Server="+appConfig['sql_server']['server']+";"
"Database="+appConfig['sql_server']['database']+";"
"UID="+appConfig['sql_server']['uid']+";"
"PWD="+appConfig['sql_server']['password']+";"
"Trusted_Connection=no;",
)
return connection
#patch('pyodbc_example.pyodbc')
class TestDatabseConnection(unittest.TestCase):
def test_getConnection(self, pyodbc_mock):
pyodbc_mock.connect.return_value = Mock()
connection = getConnection()
self.assertEqual(connection, pyodbc_mock.connect.return_value)
I'm trying to send data from my dronekit.io vehicle using flask-socket.io. Unfortunately, I got this log:
Starting copter simulator (SITL)
SITL already Downloaded and Extracted.
Ready to boot.
Connecting to vehicle on: tcp:127.0.0.1:5760
>>> APM:Copter V3.3 (d6053245)
>>> Frame: QUAD
>>> Calibrating barometer
>>> Initialising APM...
>>> barometer calibration complete
>>> GROUND START
* Restarting with stat
latitude -35.363261
>>> Exception in attribute handler for location.global_relative_frame
>>> Working outside of request context.
This typically means that you attempted to use functionality that needed
an active HTTP request. Consult the documentation on testing for
information about how to avoid this problem.
longitude 149.1652299
>>> Exception in attribute handler for location.global_relative_frame
>>> Working outside of request context.
This typically means that you attempted to use functionality that needed
an active HTTP request. Consult the documentation on testing for
information about how to avoid this problem.
Here is my code:
sample.py
from dronekit import connect, VehicleMode
from flask import Flask
from flask_socketio import SocketIO, emit
import dronekit_sitl
import time
sitl = dronekit_sitl.start_default()
connection_string = sitl.connection_string()
print("Connecting to vehicle on: %s" % (connection_string,))
vehicle = connect(connection_string, wait_ready=True)
def arm_and_takeoff(aTargetAltitude):
print "Basic pre-arm checks"
while not vehicle.is_armable:
print " Waiting for vehicle to initialise..."
time.sleep(1)
print "Arming motors"
vehicle.mode = VehicleMode("GUIDED")
vehicle.armed = True
while not vehicle.armed:
print " Waiting for arming..."
time.sleep(1)
print "Taking off!"
vehicle.simple_takeoff(aTargetAltitude)
while True:
if vehicle.location.global_relative_frame.alt>=aTargetAltitude*0.95:
print "Reached target altitude"
break
time.sleep(1)
last_latitude = 0.0
last_longitude = 0.0
last_altitude = 0.0
#vehicle.on_attribute('location.global_relative_frame')
def location_callback(self, attr_name, value):
global last_latitude
global last_longitude
global last_altitude
if round(value.lat, 6) != round(last_latitude, 6):
last_latitude = value.lat
print "latitude ", value.lat, "\n"
emit("latitude", value.lat)
if round(value.lon, 6) != round(last_longitude, 6):
last_longitude = value.lon
print "longitude ", value.lon, "\n"
emit("longitude", value.lon)
if round(value.alt) != round(last_altitude):
last_altitude = value.alt
print "altitude ", value.alt, "\n"
emit("altitude", value.alt)
app = Flask(__name__)
socketio = SocketIO(app)
if __name__ == '__main__':
socketio.run(app, host='0.0.0.0', port=5000, debug=True)
arm_and_takeoff(20)
I know because of the logs that I should not do any HTTP request inside "vehicle.on_attribute" decorator method and I should search for information on how to solve this problem but I didn't found any info about the error.
Hope you could help me.
Thank you very much,
Raniel
The emit() function by default returns an event back to the active client. If you call this function outside of a request context there is no concept of active client, so you get this error.
You have a couple of options:
indicate the recipient of the event and the namespace that you are using, so that there is no need to look them up in the context. You can do this by adding room and namespace arguments. Use '/' for the namespace if you are using the default namespace.
emit to all clients by adding broadcast=True as an argument, plus the namespace as indicated in #1.
I've been trying to create a server-process that receives an input file path and an output path from client processes asynchronously. The server does some database-reliant transformations, but for the sake of simplicity let's assume it merely puts everything to the upper case. Here is a toy example of the server:
import asyncio
import aiofiles as aiof
import logging
import sys
ADDRESS = ("localhost", 10000)
logging.basicConfig(level=logging.DEBUG,
format="%(name)s: %(message)s",
stream=sys.stderr)
log = logging.getLogger("main")
loop = asyncio.get_event_loop()
async def server(reader, writer):
log = logging.getLogger("process at {}:{}".format(*ADDRESS))
paths = await reader.read()
in_fp, out_fp = paths.splitlines()
log.debug("connection accepted")
log.debug("processing file {!r}, writing output to {!r}".format(in_fp, out_fp))
async with aiof.open(in_fp, loop=loop) as inp, aiof.open(out_fp, "w", loop=loop) as out:
async for line in inp:
out.write(line.upper())
out.flush()
writer.write(b"done")
await writer.drain()
log.debug("closing")
writer.close()
return
factory = asyncio.start_server(server, *ADDRESS)
server = loop.run_until_complete(factory)
log.debug("starting up on {} port {}".format(*ADDRESS))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
log.debug("closing server")
server.close()
loop.run_until_complete(server.wait_closed())
log.debug("closing event loop")
loop.close()
The client:
import asyncio
import logging
import sys
import random
ADDRESS = ("localhost", 10000)
MESSAGES = ["/path/to/a/big/file.txt\n",
"/output/file_{}.txt\n".format(random.randint(0, 99999))]
logging.basicConfig(level=logging.DEBUG,
format="%(name)s: %(message)s",
stream=sys.stderr)
log = logging.getLogger("main")
loop = asyncio.get_event_loop()
async def client(address, messages):
log = logging.getLogger("client")
log.debug("connecting to {} port {}".format(*address))
reader, writer = await asyncio.open_connection(*address)
writer.writelines([bytes(line, "utf8") for line in messages])
if writer.can_write_eof():
writer.write_eof()
await writer.drain()
log.debug("waiting for response")
response = await reader.read()
log.debug("received {!r}".format(response))
writer.close()
return
try:
loop.run_until_complete(client(ADDRESS, MESSAGES))
finally:
log.debug("closing event loop")
loop.close()
I activated the server and several clients at once. The server's logs:
asyncio: Using selector: KqueueSelector
main: starting up on localhost port 10000
process at localhost:10000: connection accepted
process at localhost:10000: processing file b'/path/to/a/big/file.txt', writing output to b'/output/file_79609.txt'
process at localhost:10000: connection accepted
process at localhost:10000: processing file b'/path/to/a/big/file.txt', writing output to b'/output/file_68917.txt'
process at localhost:10000: connection accepted
process at localhost:10000: processing file b'/path/to/a/big/file.txt', writing output to b'/output/file_2439.txt'
process at localhost:10000: closing
process at localhost:10000: closing
process at localhost:10000: closing
All clients print this:
asyncio: Using selector: KqueueSelector
client: connecting to localhost port 10000
client: waiting for response
client: received b'done'
main: closing event loop
The output files are created, but they remain empty. I believe they are not being flushed. Any way I can fix it?
You are missing an await before out.write() and out.flush():
import asyncio
from pathlib import Path
import aiofiles as aiof
FILENAME = "foo.txt"
async def bad():
async with aiof.open(FILENAME, "w") as out:
out.write("hello world")
out.flush()
print("done")
async def good():
async with aiof.open(FILENAME, "w") as out:
await out.write("hello world")
await out.flush()
print("done")
loop = asyncio.get_event_loop()
server = loop.run_until_complete(bad())
print(Path(FILENAME).stat().st_size) # prints 0
server = loop.run_until_complete(good())
print(Path(FILENAME).stat().st_size) # prints 11
However, I would strongly recommend trying to skip aiofiles and use regular, synchronized disk I/O, and keep asyncio for network activity:
with open(file, "w") as out: # regular file I/O
async for s in network_request(): # asyncio for slow network work. measure it!
out.write(s) # should be really quick, measure it!
i'm running into an error in trying to start up the selenium firefox driver. it seems like others have hit snags at this step, and there is no readily available solution online, so hopefully this question will be broadly helpful. it seems like firefox is failing to establish an http server interface when initiated through selenium's driver. it appears that i can run firefox from the command line with no errors.
i should specify that i am doing this via ssh login to a linux container. i'm running python2.7 on Ubuntu 14.04 LTS (GNU/Linux 3.16.3-elastic x86_64). i have the latest version of selenium (2.44) installed, and i'm using firefox 34.0. i'm using xvfb to spoof a display.
below is my code, the error logs, and some related source code.
from selenium import webdriver
d = webdriver.Firefox()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/firefox/webdriver.py", line 59, in __init__
self.binary, timeout),
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/firefox/extension_connection.py", line 47, in __init__
self.binary.launch_browser(self.profile)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/firefox/firefox_binary.py", line 66, in launch_browser
self._wait_until_connectable()
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/firefox/firefox_binary.py", line 105, in _wait_until_connectable
raise WebDriverException("Can't load the profile. Profile "
selenium.common.exceptions.WebDriverException: Message: Can't load the profile. Profile Dir: %s If you specified a log_file in the FirefoxBinary constructor, check it for details.
that error is raised here, due to a timeout:
def _wait_until_connectable(self):
"""Blocks until the extension is connectable in the firefox."""
count = 0
while not utils.is_connectable(self.profile.port):
if self.process.poll() is not None:
# Browser has exited
raise WebDriverException("The browser appears to have exited "
"before we could connect. If you specified a log_file in "
"the FirefoxBinary constructor, check it for details.")
if count == 30:
self.kill()
raise WebDriverException("Can't load the profile. Profile "
"Dir: %s If you specified a log_file in the "
"FirefoxBinary constructor, check it for details.")
count += 1
time.sleep(1)
return True
in the log_file:
tail -f logs/firefox_binary.log
1418661895753 addons.xpi DEBUG checkForChanges
1418661895847 addons.xpi DEBUG No changes found
1418661895853 addons.manager DEBUG Registering shutdown blocker for XPIProvider
1418661895854 addons.manager DEBUG Registering shutdown blocker for LightweightThemeManager
1418661895857 addons.manager DEBUG Registering shutdown blocker for OpenH264Provider
1418661895858 addons.manager DEBUG Registering shutdown blocker for PluginProvider
System JS : ERROR (null):0 - uncaught exception: 2147746065
JavaScript error: file:///tmp/tmplkLsLs/extensions/fxdriver#googlecode.com/components/driver-component.js, line 11507: NS_ERROR_NOT_AVAILABLE: Component is not available'Component is not available' when calling method: [nsIHttpServer::start]
*** Blocklist::_preloadBlocklistFile: blocklist is disabled
1418661908552 addons.manager DEBUG Registering shutdown blocker for <unnamed-provider>
one more point of information. early on, in the firefox driver initalization, socket.bind(('127.0.0.1',0)) was failing with a "can't assign requested address" error. i changed the location to (0.0.0.0,0) and edited the localhost entry in my /etc/hosts, and was able to bind that way. not sure if that could be causing the current failure though.
VV edits per louis's request VV . i specify the two lines where i change the localhost address.
def free_port():
"""
Determines a free port using sockets.
"""
free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
free_socket.bind(('0.0.0.0', 0)) # changed from 127.0.0.1
free_socket.listen(5)
port = free_socket.getsockname()[1]
free_socket.close()
return port
def is_connectable(port):
"""
Tries to connect to the server at port to see if it is running.
:Args:
- port: The port to connect.
"""
try:
socket_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_.settimeout(1)
socket_.connect(("0.0.0.0", port)) # changed again
socket_.close()
return True
except socket.error:
return False
here's the constructor from webdriver:
def __init__(self, firefox_profile=None, firefox_binary=None, timeout=30,
capabilities=None, proxy=None):
self.binary = firefox_binary
self.profile = firefox_profile
if self.profile is None:
self.profile = FirefoxProfile()
self.profile.native_events_enabled = (
self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled)
if self.binary is None:
self.binary = FirefoxBinary()
if capabilities is None:
capabilities = DesiredCapabilities.FIREFOX
if proxy is not None:
proxy.add_to_capabilities(capabilities)
RemoteWebDriver.__init__(self,
command_executor=ExtensionConnection("127.0.0.1", self.profile,
self.binary, timeout),
desired_capabilities=capabilities,
keep_alive=True)
self._is_remote = False
here's the constructor from extension_connector:
def __init__(self, host, firefox_profile, firefox_binary=None, timeout=30):
self.profile = firefox_profile
self.binary = firefox_binary
HOST = host
if self.binary is None:
self.binary = FirefoxBinary()
if HOST is None:
HOST = "127.0.0.1"
PORT = utils.free_port()
self.profile.port = PORT
self.profile.update_preferences()
self.profile.add_extension()
self.binary.launch_browser(self.profile)
_URL = "http://%s:%d/hub" % (HOST, PORT)
RemoteConnection.__init__(
self, _URL, keep_alive=True)
in posting my edits to louis's comment, i saw that my localhost issue turned up in other locations, as the host is hardcoded in twice more. i had my server master address the issue, changed everything in the source back to 127, and the problem was solved. thanks for prompting me, #louis, and i'm sorry my question wasn't more interesting. will accept my own answer after 2 days when SO allows me.