I used the solution of< https://stackoverflow.com/questions/47396372/write-opencv-frames-into-gstreamer-rtsp-server-pipeline/60580247#60580247>, but when I use multiple clients to access the rtsp stream , The error of Assertion fctx->async_lock failed at libavcodec/pthread_frame.c:155 appeared.
I am new to gstreamer, Here is the complete Python code (copied from WisdomPill's answer):
import cv2
import gi
gi.require_version('Gst', '1.0')
gi.require_version('GstRtspServer', '1.0')
from gi.repository import Gst, GstRtspServer, GObject, GLib
class SensorFactory(GstRtspServer.RTSPMediaFactory):
def __init__(self, **properties):
super(SensorFactory, self).__init__(**properties)
self.cap = cv2.VideoCapture("http://192.168.2.153/video/mjpg.cgi")
self.number_frames = 0
self.fps = 10
self.duration = 1 / self.fps * Gst.SECOND # duration of a frame in nanoseconds
self.launch_string = 'appsrc name=source is-live=true block=true format=GST_FORMAT_TIME ' \
'caps=video/x-raw,format=BGR,width=640,height=480,framerate={}/1 ' \
'! videoconvert ! video/x-raw,format=I420 ' \
'! x264enc speed-preset=ultrafast tune=zerolatency ' \
'! rtph264pay config-interval=1 name=pay0 pt=96'.format(self.fps)
def on_need_data(self, src, lenght):
if self.cap.isOpened():
ret, frame = self.cap.read()
if ret:
data = frame.tobytes()
buf = Gst.Buffer.new_allocate(None, len(data), None)
buf.fill(0, data)
buf.duration = self.duration
timestamp = self.number_frames * self.duration
buf.pts = buf.dts = int(timestamp)
buf.offset = timestamp
self.number_frames += 1
retval = src.emit('push-buffer', buf)
print ('pushed buffer, frame {}, duration {} ns, durations {} s'.format(self.number_frames, self.duration, self.duration / Gst.SECOND))
if retval != Gst.FlowReturn.OK:
print(retval)
def do_create_element(self, url):
return Gst.parse_launch(self.launch_string)
def do_configure(self, rtsp_media):
self.number_frames = 0
appsrc = rtsp_media.get_element().get_child_by_name('source')
appsrc.connect('need-data', self.on_need_data)
class GstServer(GstRtspServer.RTSPServer):
def __init__(self, **properties):
super(GstServer, self).__init__(**properties)
self.factory = SensorFactory()
self.factory.set_shared(True)
self.get_mount_points().add_factory("/test", self.factory)
self.set_service('9999')
self.attach(None)
Gst.init(None)
server = GstServer()
loop = GLib.MainLoop()
loop.run()
if you have any suggestions, I would appreciate it
Related
Although official said it can't Recover Password from .ini file, some Chinese guy said it could. But I can't make it work.
7.X, remove the first "u" in the head of password:
# python3 SecureCRTCipher.py dec hash
8.X, remove the three character such as "02:" at the begin of password:
# python3 SecureCRTCipher.py dec -v2 hash
suitable for lower than 7.X
#python3 SecureCRT-decryptpass.py 127.0.0.1.ini
They also said Error: Failed to decrypt. for me.
SecureCRTCipher.py
#!/usr/bin/env python3
import os
from Crypto.Hash import SHA256
from Crypto.Cipher import AES, Blowfish
class SecureCRTCrypto:
def __init__(self):
'''
Initialize SecureCRTCrypto object.
'''
self.IV = b'\x00' * Blowfish.block_size
self.Key1 = b'\x24\xA6\x3D\xDE\x5B\xD3\xB3\x82\x9C\x7E\x06\xF4\x08\x16\xAA\x07'
self.Key2 = b'\x5F\xB0\x45\xA2\x94\x17\xD9\x16\xC6\xC6\xA2\xFF\x06\x41\x82\xB7'
def Encrypt(self, Plaintext : str):
'''
Encrypt plaintext and return corresponding ciphertext.
Args:
Plaintext: A string that will be encrypted.
Returns:
Hexlified ciphertext string.
'''
plain_bytes = Plaintext.encode('utf-16-le')
plain_bytes += b'\x00\x00'
padded_plain_bytes = plain_bytes + os.urandom(Blowfish.block_size - len(plain_bytes) % Blowfish.block_size)
cipher1 = Blowfish.new(self.Key1, Blowfish.MODE_CBC, iv = self.IV)
cipher2 = Blowfish.new(self.Key2, Blowfish.MODE_CBC, iv = self.IV)
return cipher1.encrypt(os.urandom(4) + cipher2.encrypt(padded_plain_bytes) + os.urandom(4)).hex()
def Decrypt(self, Ciphertext : str):
'''
Decrypt ciphertext and return corresponding plaintext.
Args:
Ciphertext: A hex string that will be decrypted.
Returns:
Plaintext string.
'''
cipher1 = Blowfish.new(self.Key1, Blowfish.MODE_CBC, iv = self.IV)
cipher2 = Blowfish.new(self.Key2, Blowfish.MODE_CBC, iv = self.IV)
ciphered_bytes = bytes.fromhex(Ciphertext)
if len(ciphered_bytes) <= 8:
raise ValueError('Invalid Ciphertext.')
padded_plain_bytes = cipher2.decrypt(cipher1.decrypt(ciphered_bytes)[4:-4])
i = 0
for i in range(0, len(padded_plain_bytes), 2):
if padded_plain_bytes[i] == 0 and padded_plain_bytes[i + 1] == 0:
break
plain_bytes = padded_plain_bytes[0:i]
try:
return plain_bytes.decode('utf-16-le')
except UnicodeDecodeError:
raise(ValueError('Invalid Ciphertext.'))
class SecureCRTCryptoV2:
def __init__(self, ConfigPassphrase : str = ''):
'''
Initialize SecureCRTCryptoV2 object.
Args:
ConfigPassphrase: The config passphrase that SecureCRT uses. Leave it empty if config passphrase is not set.
'''
self.IV = b'\x00' * AES.block_size
self.Key = SHA256.new(ConfigPassphrase.encode('utf-8')).digest()
def Encrypt(self, Plaintext : str):
'''
Encrypt plaintext and return corresponding ciphertext.
Args:
Plaintext: A string that will be encrypted.
Returns:
Hexlified ciphertext string.
'''
plain_bytes = Plaintext.encode('utf-8')
if len(plain_bytes) > 0xffffffff:
raise OverflowError('Plaintext is too long.')
plain_bytes = \
len(plain_bytes).to_bytes(4, 'little') + \
plain_bytes + \
SHA256.new(plain_bytes).digest()
padded_plain_bytes = \
plain_bytes + \
os.urandom(AES.block_size - len(plain_bytes) % AES.block_size)
cipher = AES.new(self.Key, AES.MODE_CBC, iv = self.IV)
return cipher.encrypt(padded_plain_bytes).hex()
def Decrypt(self, Ciphertext : str):
'''
Decrypt ciphertext and return corresponding plaintext.
Args:
Ciphertext: A hex string that will be decrypted.
Returns:
Plaintext string.
'''
cipher = AES.new(self.Key, AES.MODE_CBC, iv = self.IV)
padded_plain_bytes = cipher.decrypt(bytes.fromhex(Ciphertext))
plain_bytes_length = int.from_bytes(padded_plain_bytes[0:4], 'little')
plain_bytes = padded_plain_bytes[4:4 + plain_bytes_length]
if len(plain_bytes) != plain_bytes_length:
raise ValueError('Invalid Ciphertext.')
plain_bytes_digest = padded_plain_bytes[4 + plain_bytes_length:4 + plain_bytes_length + SHA256.digest_size]
if len(plain_bytes_digest) != SHA256.digest_size:
raise ValueError('Invalid Ciphertext.')
if SHA256.new(plain_bytes).digest() != plain_bytes_digest:
raise ValueError('Invalid Ciphertext.')
return plain_bytes.decode('utf-8')
if __name__ == '__main__':
import sys
def Help():
print('Usage:')
print(' SecureCRTCipher.py <enc|dec> [-v2] [-p ConfigPassphrase] <plaintext|ciphertext>')
print('')
print(' <enc|dec> "enc" for encryption, "dec" for decryption.')
print(' This parameter must be specified.')
print('')
print(' [-v2] Encrypt/Decrypt with "Password V2" algorithm.')
print(' This parameter is optional.')
print('')
print(' [-p ConfigPassphrase] The config passphrase that SecureCRT uses.')
print(' This parameter is optional.')
print('')
print(' <plaintext|ciphertext> Plaintext string or ciphertext string.')
print(' NOTICE: Ciphertext string must be a hex string.')
print(' This parameter must be specified.')
print('')
def EncryptionRoutine(UseV2 : bool, ConfigPassphrase : str, Plaintext : str):
try:
if UseV2:
print(SecureCRTCryptoV2(ConfigPassphrase).Encrypt(Plaintext))
else:
print(SecureCRTCrypto().Encrypt(Plaintext))
return True
except:
print('Error: Failed to encrypt.')
return False
def DecryptionRoutine(UseV2 : bool, ConfigPassphrase : str, Ciphertext : str):
try:
if UseV2:
print(SecureCRTCryptoV2(ConfigPassphrase).Decrypt(Ciphertext))
else:
print(SecureCRTCrypto().Decrypt(Ciphertext))
return True
except:
print('Error: Failed to decrypt.')
return False
def Main(argc : int, argv : list):
if 3 <= argc and argc <= 6:
bUseV2 = False
ConfigPassphrase = ''
if argv[1].lower() == 'enc':
bEncrypt = True
elif argv[1].lower() == 'dec':
bEncrypt = False
else:
Help()
return -1
i = 2
while i < argc - 1:
if argv[i].lower() == '-v2':
bUseV2 = True
i += 1
elif argv[i].lower() == '-p' and i + 1 < argc - 1:
ConfigPassphrase = argv[i + 1]
i += 2
else:
Help()
return -1
if bUseV2 == False and len(ConfigPassphrase) != 0:
print('Error: ConfigPassphrase is not supported if "-v2" is not specified')
return -1
if bEncrypt:
return 0 if EncryptionRoutine(bUseV2, ConfigPassphrase, argv[-1]) else -1
else:
return 0 if DecryptionRoutine(bUseV2, ConfigPassphrase, argv[-1]) else -1
else:
Help()
exit(Main(len(sys.argv), sys.argv))
SecureCRT-decryptpass.py
#!/usr/bin/env python
#
# Decrypt SSHv2 passwords stored in VanDyke SecureCRT session files
# Can be found on Windows in:
# %APPDATA%\VanDyke\Config\Sessions\sessionname.ini
# Tested with version 7.2.6 (build 606) for Windows
# Eloi Vanderbeken - Synacktiv
# Decrypt SSHv2 passwords stored in VanDyke SecureCRT
# C:\>python SecureCRT-decryptpass.py -h
# usage: SecureCRT-decryptpass.py [-h] files [files ...]
#
#Tool to decrypt SSHv2 passwords in VanDyke Secure CRT session files
#
#positional arguments:
# files session file(s)
#
#optional arguments:
# -h, --help show this help message and exit
#
# C:\>python SecureCRT-decryptpass.py C:\Users\user1\AppData\Roaming\VanDyke\Config\Sessions\192.168.0.1.ini
# C:\Users\user1\AppData\Roaming\VanDyke\Config\Sessions\192.168.0.1.ini
# ssh -p 22 user#192.168.0.1 # 123456
from Crypto.Cipher import Blowfish
import argparse
import re
def decrypt(password) :
c1 = Blowfish.new('5F B0 45 A2 94 17 D9 16 C6 C6 A2 FF 06 41 82 B7'.replace(' ','').decode('hex'), Blowfish.MODE_CBC, '\x00'*8)
c2 = Blowfish.new('24 A6 3D DE 5B D3 B3 82 9C 7E 06 F4 08 16 AA 07'.replace(' ','').decode('hex'), Blowfish.MODE_CBC, '\x00'*8)
padded = c1.decrypt(c2.decrypt(password.decode('hex'))[4:-4])
p = ''
while padded[:2] != '\x00\x00' :
p += padded[:2]
padded = padded[2:]
return p.decode('UTF-16')
REGEX_HOSTNAME = re.compile(ur'S:"Hostname"=([^\r\n]*)')
REGEX_PASWORD = re.compile(ur'S:"Password"=u([0-9a-f]+)')
REGEX_PORT = re.compile(ur'D:"\[SSH2\] Port"=([0-9a-f]{8})')
REGEX_USERNAME = re.compile(ur'S:"Username"=([^\r\n]*)')
def hostname(x) :
m = REGEX_HOSTNAME.search(x)
if m :
return m.group(1)
return '???'
def password(x) :
m = REGEX_PASWORD.search(x)
if m :
return decrypt(m.group(1))
return '???'
def port(x) :
m = REGEX_PORT.search(x)
if m :
return '-p %d '%(int(m.group(1), 16))
return ''
def username(x) :
m = REGEX_USERNAME.search(x)
if m :
return m.group(1) + '#'
return ''
parser = argparse.ArgumentParser(description='Tool to decrypt SSHv2 passwords in VanDyke Secure CRT session files')
parser.add_argument('files', type=argparse.FileType('r'), nargs='+',
help='session file(s)')
args = parser.parse_args()
for f in args.files :
c = f.read().replace('\x00', '')
print f.name
print "ssh %s%s%s # %s"%(port(c), username(c), hostname(c), password(c))
I also encountered the same error and eventually found out that the pycryptodome version was not compatible, I installed the latest version and replaced it with 3.8.2 and it worked fine:
pip install pycryptodome==3.8.2
thanks for your read and help.
I have download a genebank flat file from NCBI, which contains many entries. I would like to extract three entries from each gene and make them into a table. How to realize it? Thank you much. the file from NCBI---->The table I hope to get
my friend writes it for me with python:
================================================================================
import os
import pandas as pd
from tqdm import tqdm
import sys
def search_line(gene_dict,gene_name,target,info,mode,l):
if '/{}='.format(target) in l:
if len(l.split('"')) == 3:
gene_dict[gene_name][mode].append('{} = '.format(target) + l.split('"')[1].strip('\n'))
keep_read = 0
info = []
else:
info = [l.split('"')[1].strip('\n')]
keep_read = target_list.index(target)
else:
if '"' in l:
info.append(l.strip().strip('"\n'))
if '{} = '.format(target) + ' '.join(info) not in gene_dict[gene_name][mode]:
gene_dict[gene_name][mode].append('{} = '.format(target) + ' '.join(info))
keep_read = 0
info = []
else:
info.append(l.strip())
keep_read = target_list.index(target)
return gene_dict,info,keep_read
def init_frame_dict(gene_dict,ids,mode):
frame_dict = {'gene': gene_dict[ids]['gene'], 'source': mode}
for target in target_list[1:]:
frame_dict[target] = ''
return frame_dict
def gen_frame(gene_dict,flat):
frame = []
for ids in gene_dict.keys():
for mode in gene_dict[ids].keys():
if mode not in extract_list:
continue
# print(mode)
data = gene_dict[ids][mode]
frame_dict = init_frame_dict(gene_dict, ids, mode)
for target_data in data:
for target in target_list[1:]:
if '{} = '.format(target) in target_data:
if frame_dict[target] != '':
frame.append(frame_dict)
# print(frame_dict)
frame_dict = init_frame_dict(gene_dict, ids, mode)
frame_dict[target] = target_data.split('{} = '.format(target))[1]
frame.append(frame_dict)
pd.DataFrame(frame).to_csv('{}.csv'.format(flat[:-5]))
def main():
for flat in os.listdir(path_root):
gene_dict = {}
if flat[-4:] != 'flat':
continue
with open (os.path.join(path_root,flat)) as f:
lines = f.read()
genes = lines.split('/gene=')
skip = False
for gene in tqdm(genes[1:]):
if skip:
break
lines = gene.split('\n')
gene_name = lines[0].split('"')[1]
#init paras
mode = 'init'
target = 'none'
read_mode = 0
info = []
#init dict
if gene_name not in gene_dict:
gene_dict[gene_name] = {'gene':gene_name,'mRNA':[],'ncRNA':[],'CDS':[],'misc_RNA':[],'exon':[],}
#proc lines
for l in lines:
if 'ORIGIN' in l:
skip = True
break
if ' mRNA' in l:
mode = 'mRNA'
elif ' ncRNA' in l:
mode = 'ncRNA'
elif ' CDS' in l:
mode = 'CDS'
elif ' misc_RNA' in l:
mode = 'misc_RNA'
elif ' exon' in l:
mode = 'exon'
# search_line(gene_dict, gene_name, target, info, mode, l)
if '/product=' in l and mode != 'init' or (target == 'product' and read_mode == target_list.index('product')):
target = 'product'
gene_dict,info,read_mode = search_line(gene_dict, gene_name, target, info, mode, l)
if '/protein_id=' in l and mode != 'init' or (target == 'protein_id' and read_mode == target_list.index('protein_id')):
target = 'protein_id'
gene_dict,info,read_mode = search_line(gene_dict, gene_name, target, info, mode, l)
if '/note=' in l and mode != 'init' or (target == 'note' and read_mode == target_list.index('note')):
target = 'note'
gene_dict,info,read_mode = search_line(gene_dict, gene_name, target, info, mode, l)
if '/transcript_id=' in l and mode != 'init' or (target == 'note' and read_mode == target_list.index('transcript_id')):
target = 'transcript_id'
gene_dict,info,read_mode = search_line(gene_dict, gene_name, target, info, mode, l)
gen_frame(gene_dict,flat)
if __name__ == '__main__':
target_list = ['none', 'product', 'transcript_id','protein_id','note']
extract_list = ['mRNA']
path_root = 'flats'
if not os.path.exists(path_root):
print('Please put your flat files in flats/ directory !')
sys.exit()
if len(os.listdir(path_root)) == 0:
print('No files found in flats/ directory.')
sys.exit()
main()
I am writing a website for Flask. I use a bunch of uWSGI + NGINX + Flask-Socketio. I use gevent as an asynchronous module. Errors occur during operation:
RuntimeError: You need to use the gevent-websocket server.
uwsgi_response_writev_headers_and_body_do(): Broken pipe [core/writer.c line 306] during >
Feb 23 12:57:55 toaa uwsgi[558436]: OSError: write error
I tried different configurations and also removed the async_mode='gevent' from the socketio initialization.
wsgi.py file:
from webapp import app, socketio
if __name__ == '__main__':
socketio.run(app, use_reloader=False, debug=True, log_output=True)
project.ini:
[uwsgi]
module = wsgi:app
master = true
gevent = 1024
gevent-monkey-patch = true
buffer-size=32768 # optionally
socket = /home/sammy/projectnew/projectnew.sock
socket-timeout = 240
chmod-socket = 664
vacuum = true
die-on-term = true
webapp/__init__.py for application app:
from gevent import monkey
monkey.patch_all()
import grpc.experimental.gevent
grpc.experimental.gevent.init_gevent()
from flask import Flask, session, request
from config import DevelopConfig, MqttConfig, MailConfig, ProductionConfig
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_mail import Mail
from flask_script import Manager
from flask_socketio import SocketIO
# from flask_mqtt import Mqtt
from flask_login import LoginManager
from flask_babel import Babel
from flask_babel_js import BabelJS
from flask_babel import lazy_gettext as _l
from apscheduler.schedulers.gevent import GeventScheduler
# from celery import Celery
app = Flask(__name__)
app.config.from_object(ProductionConfig)
app.config.from_object(MqttConfig)
app.config.from_object(MailConfig)
db = SQLAlchemy(app)
migrate = Migrate(app, db)
mail = Mail(app)
manager = Manager(app, db)
login_manager = LoginManager(app)
login_manager.login_view = 'auth'
login_manager.login_message = _l("Необходимо авторизоваться для доступа к закрытой странице")
login_manager.login_message_category = "error"
# celery = Celery(app.name, broker=Config.CELERY_BROKER_URL)
# celery.conf.update(app.config)
scheduler = GeventScheduler()
# socketio = SocketIO(app) - Production Version
socketio = SocketIO(app, async_mode='gevent')
babel = Babel(app)
babeljs = BabelJS(app=app, view_path='/translations/')
import webapp.views
#babel.localeselector
def get_locale():
# if the user has set up the language manually it will be stored in the session,
# so we use the locale from the user settings
try:
language = session['language']
except KeyError:
language = None
if language is not None:
print(language)
return language
return request.accept_languages.best_match(app.config['LANGUAGES'].keys())
from webapp import models
if __name__ == "__main__":
manager.run()
The class in which the socket itself is used (mqtt.py):
from webapp import socketio, app
from flask import request
from flask_mqtt import Mqtt
from flask_babel import lazy_gettext as _l
from webapp.tasks import SchedulerTask
from webapp import translate_state_gate as tr_msg
import json
import copy
import logging
mqtt = Mqtt(app)
logger = logging.getLogger('flask.flask_mqtt')
logger.disabled = True
class MqttTOAA(object):
type_topic = ["/Control", "/Data"]
m_request_state = {"comm": "3"}
m_start = {"Gate": "Start"}
m_stop = {"Gate": "Stop"}
qos_request = 1
qos_sub = 2
struct_state_devices = None
POOL_TIME = 2
end_publish = None
devices = None
schedulers_list = list()
sch_task = None
sid_mqtt = None
code_list = list()
def __init__(self, devices, lang):
mqtt._connect()
self.devices = devices
self.sch_task = SchedulerTask()
if lang not in app.config['LANGUAGES'].keys():
lang = 'ru'
self.dict_gate = {"dict_state_button": {'con_Clos': tr_msg.MessageGate.t_message[lang]["f_open"],
'con_Open': tr_msg.MessageGate.t_message[lang]["f_close"],
"fl_OpenClos": (tr_msg.MessageGate.t_message[lang]["f_continue"],
tr_msg.MessageGate.t_message[lang]["f_stop"],
tr_msg.MessageGate.t_message[lang]["f_abort"])},
"dict_state_text": {tr_msg.MessageGate.t_message[lang]["f_open"]:\
tr_msg.MessageGate.t_message[lang]["ps_close"],
tr_msg.MessageGate.t_message[lang]["f_close"]:\
tr_msg.MessageGate.t_message[lang]["ps_open"],
tr_msg.MessageGate.t_message[lang]["f_continue"]:\
tr_msg.MessageGate.t_message[lang]["ps_stop"],
tr_msg.MessageGate.t_message[lang]["f_abort"]:\
tr_msg.MessageGate.t_message[lang]["pr_close"],
tr_msg.MessageGate.t_message[lang]["f_stop"]:\
(tr_msg.MessageGate.t_message[lang]["pr_open"],
tr_msg.MessageGate.t_message[lang]["pr_close"],
tr_msg.MessageGate.t_message[lang]["pr_move"])},
"dict_type_element": {"button": u'', "text": u'', "device_code": u'', },
"state_gate": {},
"position": {"state": u'', "stop": False},
"reverse": False,
}
self.close_msg = tr_msg.MessageGate.t_message[lang]["pr_close"]
self.open_msg = tr_msg.MessageGate.t_message[lang]["pr_open"]
self.create_devices_dict()
self.handle_mqtt_connect()
self.mqtt_onmessage = mqtt.on_message()(self._handle_mqtt_message)
self.mqtt_onlog = mqtt.on_log()(self._handle_logging)
self.socketio_error = socketio.on_error()(self._handle_error)
self.handle_change_state = socketio.on('change_state')(self._handle_change_state)
self.handle_on_connect = socketio.on('connect')(self._handle_on_connect)
self.handle_unsubscribe_all = socketio.on('unsubscribe_all')(self._handle_unsubscribe_all)
def _handle_on_connect(self):
self.sid_mqtt = request.sid
def handle_mqtt_connect(self):
task = None
for dev in self.devices:
if dev.device_code not in self.code_list:
mqtt.subscribe("BK" + dev.device_code + self.type_topic[1], self.qos_sub)
self.code_list.append(dev.device_code)
task = self.sch_task.add_scheduler_publish(dev.device_code,
mqtt,
"BK" + dev.device_code +
self.type_topic[0],
self.m_request_state,
self.qos_request,
self.POOL_TIME)
if task is not None:
self.schedulers_list.append(task)
if len(self.schedulers_list) > 0:
self.sch_task.start_schedulers()
self.code_list.clear()
#staticmethod
def _handle_error():
print(request.event["message"]) # "my error event"
print(request.event["args"]) # (data,)
#staticmethod
def _handle_unsubscribe_all():
mqtt.unsubscribe_all()
def _handle_change_state(self, code):
print(code)
# print(self.struct_state_devices[code])
message = None
if code is not None:
try:
type_g = self.struct_state_devices[code]["state_gate"]
if type_g["fl_OpenClos"] == 1:
message = self.m_stop
else:
if self.struct_state_devices[code]["reverse"] is True:
if self.struct_state_devices[code]["position"]["state"] == self.close_msg:
message = self.m_stop
self.struct_state_devices[code]["position"]["state"] = self.open_msg
else:
message = self.m_start
else:
message = self.m_start
print("Msg:" + str(message))
except Exception as ex:
print(ex)
if message is not None:
mqtt.publish("BK" + code + self.type_topic[0], json.dumps(message), self.qos_request)
else:
print("Error change state " + code)
def _handle_mqtt_message(self, client, userdata, message):
# print("Get message")
# print(self.struct_state_devices)
data = dict(
topic=message.topic,
payload=message.payload.decode(),
qos=message.qos,
)
try:
data = json.loads(data['payload'])
self.gate_msg(data)
except Exception as ex:
print("Exception: " + str(ex))
#staticmethod
def _handle_logging(self, client, userdata, level, buf):
print(level, buf)
pass
def create_devices_dict(self):
if self.struct_state_devices is None:
self.struct_state_devices = dict()
for dev in self.devices:
self.struct_state_devices[dev.device_code] = self.dict_gate.copy()
if dev.typedev.reverse:
self.struct_state_devices[dev.device_code]['reverse'] = True
def gate_msg(self, data):
k = ""
code = data["esp_id"][2:]
dict_dev = copy.deepcopy(self.struct_state_devices[code])
dict_dev["state_gate"] = data.copy()
try:
if dict_dev["state_gate"]["con_Clos"] == 0: # ворота закрыты
# print("1")
k = "con_Clos"
dict_dev["position"]["state"] = k
dict_dev["position"]["stop"] = False
elif dict_dev["state_gate"]["con_Open"] == 0: # ворота открыты
# print("2")
k = "con_Open"
dict_dev["position"]["state"] = k
dict_dev["position"]["stop"] = False
elif dict_dev["state_gate"]["fl_OpenClos"] == 0:
# print("3")
k = "fl_OpenClos"
# обратный ход ворот при закрытии
if dict_dev["position"]["state"] == self.close_msg and dict_dev["reverse"] is True:
# print("4")
k1 = 1
k2 = 0
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
dict_dev["position"]["stop"] = False
else:
# print("5")
k1 = 0
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]]
dict_dev["position"]["stop"] = True
elif dict_dev["state_gate"]["fl_OpenClos"] == 1:
# print("6")
k = "fl_OpenClos"
if len(dict_dev["position"]["state"]) == 0:
# print("7")
k1 = 1
k2 = 2
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
elif dict_dev["position"]["state"] == "con_Clos" or \
dict_dev["position"]["state"] == self.open_msg:
if dict_dev["position"]["stop"]:
# print("8")
k1 = 1
k2 = 1
dict_dev["position"]["stop"] = False
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
else:
# print("9")
k1 = 1
k2 = 0
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
elif dict_dev["position"]["state"] == "con_Open" or \
dict_dev["position"]["state"] == self.close_msg:
if dict_dev["reverse"]:
# print("10")
k1 = 2
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]]
else:
if dict_dev["position"]["stop"]:
# print("11")
k1 = 1
k2 = 0
dict_dev["position"]["stop"] = False
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
else:
# print("12")
k1 = 1
k2 = 1
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k][k1]][k2]
if dict_dev["position"]["state"] != dict_dev["dict_type_element"]["text"]:
# print("13")
dict_dev["position"]["state"] = dict_dev["dict_type_element"]["text"]
if k == "fl_OpenClos":
dict_dev["dict_type_element"]["button"] = dict_dev["dict_state_button"][k][k1]
else:
dict_dev["dict_type_element"]["button"] = dict_dev["dict_state_button"][k]
dict_dev["dict_type_element"]["text"] = \
dict_dev["dict_state_text"][dict_dev["dict_state_button"][k]]
except Exception as ex:
print("Exception (gate_msg): " + str(ex))
dict_dev["dict_type_element"]["device_code"] = data["esp_id"][2:]
dict_dev["dict_type_element"]["temp"] = data["temp_1"]
dict_dev["dict_type_element"]["button"] = copy.deepcopy(str(dict_dev["dict_type_element"]["button"]))
dict_dev["dict_type_element"]["text"] = copy.deepcopy(str(dict_dev["dict_type_element"]["text"]))
self.struct_state_devices[code] = copy.deepcopy(dict_dev)
socketio.emit('mqtt_message', data=dict_dev["dict_type_element"], room=self.sid_mqtt)
# print(dict_dev["state_gate"]["esp_id"] + str(dict_dev["dict_type_element"]))
When you use uWSGI, the async_mode should be gevent_uwsgi:
socketio = SocketIO(app, async_mode='gevent_uwsgi')
I have been trying to download all the YouTube comments on popular videos using python requests, but it has been throwing up the following error after about a quarter of the total comments:
{'error': {'code': 400, 'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'errors': [{'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'domain': 'youtube.commentThread', 'reason': 'processingFailure', 'location': 'body', 'locationType': 'other'}]}}
I found this thread detailing the same issue, and it seems that it is not possible to download all the comments on popular videos.
This is my code:
import argparse
import urllib
import requests
import json
import time
start_time = time.time()
class YouTubeApi():
YOUTUBE_COMMENTS_URL = 'https://www.googleapis.com/youtube/v3/commentThreads'
comment_counter = 0
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
def format_comments(self, results, likes_required):
comments_list = []
try:
for item in results["items"]:
comment = item["snippet"]["topLevelComment"]
likes = comment["snippet"]["likeCount"]
if likes < likes_required:
continue
author = comment["snippet"]["authorDisplayName"]
text = comment["snippet"]["textDisplay"]
str = "Comment by {}:\n \"{}\"\n\n".format(author, text)
str = str.encode('ascii', 'replace').decode()
comments_list.append(str)
self.comment_counter += 1
print("Comments downloaded:", self.comment_counter, end="\r")
except(KeyError):
print(results)
return comments_list
def get_video_comments(self, video_id, likes_required):
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
if self.comment_counter <= 900000:
key = self.key_list[0]
elif self.comment_counter <= 1800000:
key = self.key_list[1]
elif self.comment_counter <= 2700000:
key = self.key_list[2]
elif self.comment_counter <= 3600000:
key = self.key_list[3]
elif self.comment_counter <= 4500000:
key = self.key_list[4]
params = {
'part': 'snippet,replies',
'maxResults': 100,
'videoId': video_id,
'textFormat': 'plainText',
'key': key
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
try:
#data = self.openURL(self.YOUTUBE_COMMENTS_URL, params)
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list = []
commments_list += self.format_comments(results, likes_required)
while nextPageToken:
params.update({'pageToken': nextPageToken})
try:
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError as c:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list += self.format_comments(results, likes_required)
return commments_list
def get_video_id_list(self, filename):
try:
with open(filename, 'r') as file:
URL_list = file.readlines()
except FileNotFoundError:
exit("File \"" + filename + "\" not found")
list = []
for url in URL_list:
if url == "\n": # ignore empty lines
continue
if url[-1] == '\n': # delete '\n' at the end of line
url = url[:-1]
if url.find('='): # get id
id = url[url.find('=') + 1:]
list.append(id)
else:
print("Wrong URL")
return list
def main():
yt = YouTubeApi()
parser = argparse.ArgumentParser(add_help=False, description=("Download youtube comments from many videos into txt file"))
required = parser.add_argument_group("required arguments")
optional = parser.add_argument_group("optional arguments")
here: https://console.developers.google.com/apis/credentials")
optional.add_argument("--likes", '-l', help="The amount of likes a comment needs to be saved", type=int)
optional.add_argument("--input", '-i', help="URL list file name")
optional.add_argument("--output", '-o', help="Output file name")
optional.add_argument("--help", '-h', help="Help", action='help')
args = parser.parse_args()
# --------------------------------------------------------------------- #
likes = 0
if args.likes:
likes = args.likes
input_file = "URL_list.txt"
if args.input:
input_file = args.input
output_file = "Comments.txt"
if args.output:
output_file = args.output
list = yt.get_video_id_list(input_file)
if not list:
exit("No URLs in input file")
try:
vid_counter = 0
with open(output_file, "a") as f:
for video_id in list:
vid_counter += 1
print("Downloading comments for video ", vid_counter, ", id: ", video_id, sep='')
comments = yt.get_video_comments(video_id, likes)
if comments:
for comment in comments:
f.write(comment)
print('\nDone!')
except KeyboardInterrupt:
exit("User Aborted the Operation")
# --------------------------------------------------------------------- #
if __name__ == '__main__':
main()
The next best method would be to randomly sample them. Does anyone know if this is possible with the API V3?
Even if the API returns a processingFailure error, you could still catch that (or any other API error for that matter) for to terminate gracefully your pagination loop. This way your script will provide the top-level comments that it fetched from of the API prior to the occurrence of the first API error.
The error response provided by the YouTube Data API is (usually) of the following form:
{
"error": {
"errors": [
{
"domain": <string>,
"reason": <string>,
"message": <string>,
"locationType": <string>,
"location": <string>
}
],
"code": <integer>,
"message": <string>
}
}
Hence, you could have defined the following function:
def is_error_response(response):
error = response.get('error')
if error is None:
return False
print("API Error: "
f"code={error['code']} "
f"domain={error['errors'][0]['domain']} "
f"reason={error['errors'][0]['reason']} "
f"message={error['errors'][0]['message']!r}")
return True
that you'll invoke after each statement of form results = comments_data.json(). In case of the first occurrence of that statement, you'll have:
results = comments_data.json()
if is_error_response(results):
return []
nextPageToken = results.get("nextPageToken")
For the second instance of that statement:
results = comments_data.json()
if is_error_response(results):
return comments_list
nextPageToken = results.get("nextPageToken")
Notice that the function is_error_response above prints out an error message on stdout in case its argument in an API error response; this is for the purpose of having the user of your script informed about the API call failure.
I have a strange issue with my python monitoring script:-
I've written a script with a number of alerts for any server. In that I have a function that gathers Network bytes/sec in and out.
Now the issue is when I print the alert outside my mail function it prints the current output, but for some reason when it triggers the mail for the alert, the mail body is empty. If I trigger the mail with another alert which isn't in the Network function it works properly.
Also is there a way to get smtplib to use port 587 instead of 465, any pointers on formatting the alert would be appreciated too.
Please find my script below:-
#!/usr/bin/env python3
#Module psutil needs to be installed via pip3 first.
#Python script to Monitor Server Resources.
import time
import psutil
import smtplib
from email.message import EmailMessage
project_and_instance_name = 'test-stage' #Edit the name of the project name and environment
sender = '<sender email>' #Email Address of the sender
receivers = ['recepient email'] #comma seperated list of recipients enclosed in ''
cpu_thresh = 50.0
cpu_pct = psutil.cpu_percent(interval=1)
if cpu_pct >= cpu_thresh:
cpu_alert = "CPU Warning, CPU at ",cpu_pct, "percent"
else:
cpu_alert = ""
mem = psutil.virtual_memory()
mem_thresh = 1024 * 1024 * 1024 #change the end value to choose the amount of MB
if mem_thresh >= mem.available:
mem_alert = "Memory Usage Warning only", round((mem.available /1024 /1024), 2), "MB available"
else:
mem_alert = ""
partition1 = '/'
disk1 = psutil.disk_usage(partition1)
disk_thresh = 85.0
if disk_thresh <= disk1[3]:
disk_alert = f"Root volume usage warning {disk1[3]} % used"
else:
disk_alert = ""
def net_usage(inf = "eth0"): #change the inf variable according to the interface
global net_in_alert
global net_out_alert
net_in_ps1 = psutil.net_io_counters(pernic=True, nowrap=True)[inf]
net_in_1 = net_in_ps1.bytes_recv
net_out_1 = net_in_ps1.bytes_sent
time.sleep(1)
net_in_ps2 = psutil.net_io_counters(pernic=True, nowrap=True)[inf]
net_in_2 = net_in_ps2.bytes_recv
net_out_2 = net_in_ps2.bytes_sent
net_in_res = round((net_in_2 - net_in_1) /1024 /1024, 2)
net_out_res = round((net_out_2 - net_out_1) /1024 /1024, 2)
net_in_thresh = 1.5
net_out_thresh = 1.5
if net_in_res >= net_in_thresh:
net_in_alert = f"Current net-usage:IN: {net_in_res} MB/s"
else:
net_in_alert = ""
if net_out_res <= net_out_thresh:
net_out_alert = f"Current net-usage:OUT: {net_out_res} MB/s"
else:
net_out_alert = ""
net_usage()
message_list = []
if cpu_alert == "" :
pass
else:
message_list.append(cpu_alert)
if mem_alert == "" :
pass
else:
message_list.append(mem_alert)
if disk_alert == "" :
pass
else:
message_list.append(disk_alert)
if net_in_alert == "" :
pass
else:
message_list.append(net_in_alert)
if net_out_alert == "" :
pass
else:
message_list.append(net_out_alert)
msg = '\n'.join(message_list)
print(msg)
def alerts():
server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
server.login(sender, "<password>")
server.sendmail(sender,receivers,msg)
if msg == "":
pass
else:
alerts()
Got the answer by changing the SMTP format for any who are stuck here's the code:-
#!/usr/bin/env python3
#Module psutil needs to be installed via pip3 first.
#Python script to Monitor Server Resources.
import time
import psutil
import smtplib, ssl
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
project_and_instance_name = 'test-stage' #Edit the name of the project name and environment
sender = '<senders email>' #Email Address of the sender
receivers = ['recepient email'] #comma seperated list of recipients enclosed in ''
cpu_thresh = 50.0
cpu_pct = psutil.cpu_percent(interval=1)
if cpu_pct >= cpu_thresh:
cpu_alert = "CPU Warning, CPU at ",cpu_pct, "percent"
else:
cpu_alert = ""
mem = psutil.virtual_memory()
mem_thresh = 1024 * 1024 * 1024 #change the end value to choose the amount of MB
if mem_thresh >= mem.available:
mem_alert = "Memory Usage Warning only", round((mem.available /1024 /1024), 2), "MB available"
else:
mem_alert = ""
partition1 = '/'
disk1 = psutil.disk_usage(partition1)
disk_thresh = 85.0
if disk_thresh <= disk1[3]:
disk_alert = f"Root volume usage warning {disk1[3]} % used"
else:
disk_alert = ""
def net_usage(inf = "eth0"): #change the inf variable according to the interface
global net_in_alert
global net_out_alert
net_in_ps1 = psutil.net_io_counters(pernic=True, nowrap=True)[inf]
net_in_1 = net_in_ps1.bytes_recv
net_out_1 = net_in_ps1.bytes_sent
time.sleep(1)
net_in_ps2 = psutil.net_io_counters(pernic=True, nowrap=True)[inf]
net_in_2 = net_in_ps2.bytes_recv
net_out_2 = net_in_ps2.bytes_sent
net_in_res = round((net_in_2 - net_in_1) /1024 /1024, 2)
net_out_res = round((net_out_2 - net_out_1) /1024 /1024, 2)
net_in_thresh = 1.5
net_out_thresh = 1.5
if net_in_res >= net_in_thresh:
net_in_alert = f"Current net-usage:IN: {net_in_res} MB/s"
else:
net_in_alert = ""
if net_out_res >= net_out_thresh:
net_out_alert = f"Current net-usage:OUT: {net_out_res} MB/s"
else:
net_out_alert = ""
net_usage()
message_list = []
if cpu_alert == "" :
pass
else:
message_list.append(cpu_alert)
if mem_alert == "" :
pass
else:
message_list.append(mem_alert)
if disk_alert == "" :
pass
else:
message_list.append(disk_alert)
if net_in_alert == "" :
pass
else:
message_list.append(net_in_alert)
if net_out_alert == "" :
pass
else:
message_list.append(net_out_alert)
msg = '\n'.join(message_list)
print(msg)
def alerts():
msg_template = MIMEMultipart()
msg_template['From'] = sender
msg_template['To'] = ', '.join(receivers)
msg_template['Subject'] = f"{project_and_instance_name} Alert"
msg_template.attach(MIMEText(msg, 'plain'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.ehlo()
server.starttls()
server.ehlo()
server.login(sender, "<password>")
server.sendmail(sender,receivers,msg_template.as_string())
server.quit()
if msg == "":
pass
else:
alerts()
Something helpful for me in understanding this answer -
receivers is a list of emails
receivers = ['email1', 'email2']
message template ["To"] is a joined string
msg_template['To'] = ', '.join(receivers)
server.sendmail() is the list of emails
server.sendmail(sender,receivers,msg_template.as_string())