I am writing a Let's Encrypt ACME client in Elixir+escript. The basics seem to be almost done except whenever I submit I get an error back that the key is too large. I have included full source code as well as the results from IO.inspect.
Code:
defmodule NginxDockerCerts do
#base "https://acme-staging.api.letsencrypt.org"
def main(_args) do
{ :ok, %{ body: body } } = HTTPoison.get(#base <> "/directory")
endpoints = Poison.decode!(body)
response = send_request(
endpoints["new-reg"],
%{ resource: "new-reg", contact: ["mailto:some#dude.com"] } )
IO.inspect(response)
end
defp send_request(url, payload) do
[private_pem] = File.read!("acme_key/private.key") |> :public_key.pem_decode
private_key = :public_key.pem_entry_decode(private_pem)
IO.inspect(private_key)
header = generate_header(private_key)
request = %{
payload: b64(payload),
header: header,
protected: Map.merge(header, %{ nonce: nonce() }) |> b64
}
signature = "#{request[:payload]}.#{request[:protected]}"
|> :public_key.sign(:sha256, private_key)
|> b64
request = Map.put(request, :signature, signature)
IO.inspect Poison.encode!(request)
HTTPoison.post(url, Poison.encode!(request))
end
def nonce do
{ :ok, %{ headers: le_headers } } = HTTPoison.head(#base <> "/directory")
%{ "Replay-Nonce" => ret } = Enum.into(le_headers, %{})
ret
end
# From the erlang docs:
# Key = public_key:pem_entry_decode(RSAEntry, "abcd1234").
# 'RSAPrivateKey'{version = 'two-prime',
# modulus = 1112355156729921663373...2737107,
# publicExponent = 65537,
# privateExponent = 58064406231183...2239766033,
# prime1 = 11034766614656598484098...7326883017,
# prime2 = 10080459293561036618240...77738643771,
# exponent1 = 77928819327425934607...22152984217,
# exponent2 = 36287623121853605733...20588523793,
# coefficient = 924840412626098444...41820968343,
# otherPrimeInfos = asn1_NOVALUE}
defp generate_header(tuple) do
modulus = elem(tuple, 2) |> Integer.to_string |> b64
exponent = elem(tuple, 3) |> Integer.to_string |> b64
%{
alg: "RS256",
jwk: %{ e: exponent, kty: "RSA", n: modulus }
}
end
defp b64(map) when is_map(map) do
map |> Poison.encode! |> b64
end
defp b64(str) when is_binary(str) do
Base.url_encode64(str, padding: false)
end
end
IO.inspect(private_key):
{:RSAPrivateKey, :"two-prime",
26481417694003873198106692499951294632119767655743036419434387596516366789190701823748669795645320777784137996798720069366288226752787622864701474496474449030506709232606243234272615772772466937031758607350665927756585455359455675325206612466347661342624412526201816026642296038870194629664917766048486662184822018742110266572261560976195129025393833146676682160704214817952205520453818572377683628606127111066471430372867661099887369197771195456883038219184710367619732291239260771178775267372942936652670907351820071078279876378927060202383261106965701867067167942374432028957977689700100770593457365162852102994011,
65537,
17914363694762679375590280482557038734074338145129736174308002717419978357245064091366667308389692794651262006012958394118656461740464764327121166238936702897104154200338706284869255844574321706109571964796859390986591027409882468918977602937203771390592387954119924829351148128283413627511087311865963595647490630788471521560987974810414561435248783963374268078499217692971351713183762668005601250077516937260913883929834502151959055904643039470485089018275006363270281504396441774935380614604745759736941304083977023940167676210942357786129557579261694171244151039301867300063202512362347353203092947886172660735393,
178446771914665404540695905367251634656603255428643793397732858767097609718259368256180025741118899743098924191892492358642536804962239144370164933768905841306132489144152437556760519068703335707439676521498046102518700677420206003309978859591990308609070226460982935297571405985650480509898548679141561783409,
148399533428755367732844203564110326749419930547476575029427037695010307782491560655098073907520278149390027076214510851700665327903331461330257316301355625227675837876554040085530270141360998128496771695290961076657330075330485591211033119775343431677271581646470921917040927840058534430663755569847957098379,
173496648605071408699339341045453818586204782529834351152938645769270680587115197307101867039967578203618114203933621017140604584820619131790019979779680353116338191796637764448772807948772568899131006122851732603054596464051422956877924576259853677682245842193987696327661510566545050393063851309736807837969,
104457312306054806733354226690510299270297554298879135185963633915322955098861989175280059316535971303378417368095146254784372068320316518037537578135981756067227887713555921436525884491068010508044075485229218997318755721273030971941897994237703401844823768755563271113340785238716148920776808645385905807487,
167919839448914714974178630755108847455535988232500360818424531493548836346962617553207612824957810666270161293456074716336654588650394703385074422991192683242596666041121521405849960615014764544853974028554205453198599086267359147206158727144128096959265897638538959277458154234233457521650291895863656575525,
:asn1_NOVALUE}
IO.inspect Poison.encode!(request):
"{\"signature\":\"bzWgP-iSDaFG1tl0nbm4jfWEeQbwpfPEGqKdyslskweSpZCMXdgP7NPCAYlbX8W5qExPA9wGtI0uR2tfRA-ALE6prvRxAwzh3BfwPNscwgXzTNUy79KLkW78eFMLrUvU8shtkwxXlmmPCxGd2PAMM5vXUHM_ovVhTHKqojdy5ECzpw6u03k2wJZzLIn68o-G4ZuOnThl2HzDky29CCYSZnkDk3uE2L5rt1fyqG2II6DhKYKlXB8fittUSJk4xDr7ufu4kKOlMqXB2JyfR6kpEVigqUsjSzAWCzdP3WXgk-M8A2ElqGuwzXgXiXiOMD04SIOCSxfe1Qu6KFeQBeEgPw\",\"protected\":\"eyJub25jZSI6IkVfUVZLVjlNVklRT29odjRnbG51VDEyem1NNE5sb0tBRUZzUFl6WWJuUXMiLCJqd2siOnsibiI6Ik1qWTBPREUwTVRjMk9UUXdNRE00TnpNeE9UZ3hNRFkyT1RJME9UazVOVEV5T1RRMk16SXhNVGszTmpjMk5UVTNORE13TXpZME1UazBNelF6T0RjMU9UWTFNVFl6TmpZM09Ea3hPVEEzTURFNE1qTTNORGcyTmprM09UVTJORFV6TWpBM056YzNPRFF4TXpjNU9UWTNPVGczTWpBd05qa3pOall5T0RneU1qWTNOVEkzT0RjMk1qSTROalEzTURFME56UTBPVFkwTnpRME5Ea3dNekExTURZM01Ea3lNekkyTURZeU5ETXlNelF5TnpJMk1UVTNOekkzTnpJME5qWTVNemN3TXpFM05UZzJNRGN6TlRBMk5qVTVNamMzTlRZMU9EVTBOVFV6TlRrME5UVTJOelV6TWpVeU1EWTJNVEkwTmpZek5EYzJOakV6TkRJMk1qUTBNVEkxTWpZeU1ERTRNVFl3TWpZMk5ESXlPVFl3TXpnNE56QXhPVFEyTWprMk5qUTVNVGMzTmpZd05EZzBPRFkyTmpJeE9EUTRNakl3TVRnM05ESXhNVEF5TmpZMU56SXlOakUxTmpBNU56WXhPVFV4TWprd01qVXpPVE00TXpNeE5EWTJOelkyT0RJeE5qQTNNRFF5TVRRNE1UYzVOVEl5TURVMU1qQTBOVE00TVRnMU56SXpOemMyT0RNMk1qZzJNRFl4TWpjeE1URXdOalkwTnpFME16QXpOekk0TmpjMk5qRXdPVGs0T0Rjek5qa3hPVGMzTnpFeE9UVTBOVFk0T0RNd016Z3lNVGt4T0RRM01UQXpOamMyTVRrM016SXlPVEV5TXpreU5qQTNOekV4TnpnM056VXlOamN6TnpJNU5ESTVNelkyTlRJMk56QTVNRGN6TlRFNE1qQXdOekV3TnpneU56azROell6TnpnNU1qY3dOakF5TURJek9ETXlOakV4TURZNU5qVTNNREU0Tmpjd05qY3hOamM1TkRJek56UTBNekl3TWpnNU5UYzVOemMyT0RrM01EQXhNREEzTnpBMU9UTTBOVGN6TmpVeE5qSTROVEl4TURJNU9UUXdNVEUiLCJrdHkiOiJSU0EiLCJlIjoiTmpVMU16YyJ9LCJhbGciOiJSUzI1NiJ9\",\"payload\":\"eyJyZXNvdXJjZSI6Im5ldy1yZWciLCJjb250YWN0IjpbIm1haWx0bzpzb21lQGR1ZGUuY29tIl19\",\"header\":{\"jwk\":{\"n\":\"MjY0ODE0MTc2OTQwMDM4NzMxOTgxMDY2OTI0OTk5NTEyOTQ2MzIxMTk3Njc2NTU3NDMwMzY0MTk0MzQzODc1OTY1MTYzNjY3ODkxOTA3MDE4MjM3NDg2Njk3OTU2NDUzMjA3Nzc3ODQxMzc5OTY3OTg3MjAwNjkzNjYyODgyMjY3NTI3ODc2MjI4NjQ3MDE0NzQ0OTY0NzQ0NDkwMzA1MDY3MDkyMzI2MDYyNDMyMzQyNzI2MTU3NzI3NzI0NjY5MzcwMzE3NTg2MDczNTA2NjU5Mjc3NTY1ODU0NTUzNTk0NTU2NzUzMjUyMDY2MTI0NjYzNDc2NjEzNDI2MjQ0MTI1MjYyMDE4MTYwMjY2NDIyOTYwMzg4NzAxOTQ2Mjk2NjQ5MTc3NjYwNDg0ODY2NjIxODQ4MjIwMTg3NDIxMTAyNjY1NzIyNjE1NjA5NzYxOTUxMjkwMjUzOTM4MzMxNDY2NzY2ODIxNjA3MDQyMTQ4MTc5NTIyMDU1MjA0NTM4MTg1NzIzNzc2ODM2Mjg2MDYxMjcxMTEwNjY0NzE0MzAzNzI4Njc2NjEwOTk4ODczNjkxOTc3NzExOTU0NTY4ODMwMzgyMTkxODQ3MTAzNjc2MTk3MzIyOTEyMzkyNjA3NzExNzg3NzUyNjczNzI5NDI5MzY2NTI2NzA5MDczNTE4MjAwNzEwNzgyNzk4NzYzNzg5MjcwNjAyMDIzODMyNjExMDY5NjU3MDE4NjcwNjcxNjc5NDIzNzQ0MzIwMjg5NTc5Nzc2ODk3MDAxMDA3NzA1OTM0NTczNjUxNjI4NTIxMDI5OTQwMTE\",\"kty\":\"RSA\",\"e\":\"NjU1Mzc\"},\"alg\":\"RS256\"}}"
IO.inspect(response):
{:ok,
%HTTPoison.Response{body: "{\n \"type\": \"urn:acme:error:malformed\",\n \"detail\": \"Key too large: 4934 \\u003e 4096\",\n \"status\": 400\n}",
headers: [{"Server", "nginx"}, {"Content-Type", "application/problem+json"},
{"Content-Length", "104"},
{"Boulder-Request-Id", "5BbL5f23yYxol0_rHyC7OT88PF5BJle7lY970szFsqg"},
{"Replay-Nonce", "DFPac8kWo7OoL5LSHUI6CZr5nHjZzyeA64eXQdcniVg"},
{"Expires", "Sat, 04 Feb 2017 05:12:57 GMT"},
{"Cache-Control", "max-age=0, no-cache, no-store"}, {"Pragma", "no-cache"},
{"Date", "Sat, 04 Feb 2017 05:12:57 GMT"}, {"Connection", "close"}],
status_code: 400}}
You're converting the integers in RSAPrivateKey to binary using String.to_integer, which creates an ASCII representation of the digits in base 10. You need convert every byte in the number into a binary with the appropriate total number of bytes.
One way to do this is to specify a size using the <<>> syntax:
iex(1)> n = 0x12345678
305419896
iex(2)> <<n::size(32)>> # or just <<n::32>>
<<18, 52, 86, 120>>
Related
Although official said it can't Recover Password from .ini file, some Chinese guy said it could. But I can't make it work.
7.X, remove the first "u" in the head of password:
# python3 SecureCRTCipher.py dec hash
8.X, remove the three character such as "02:" at the begin of password:
# python3 SecureCRTCipher.py dec -v2 hash
suitable for lower than 7.X
#python3 SecureCRT-decryptpass.py 127.0.0.1.ini
They also said Error: Failed to decrypt. for me.
SecureCRTCipher.py
#!/usr/bin/env python3
import os
from Crypto.Hash import SHA256
from Crypto.Cipher import AES, Blowfish
class SecureCRTCrypto:
def __init__(self):
'''
Initialize SecureCRTCrypto object.
'''
self.IV = b'\x00' * Blowfish.block_size
self.Key1 = b'\x24\xA6\x3D\xDE\x5B\xD3\xB3\x82\x9C\x7E\x06\xF4\x08\x16\xAA\x07'
self.Key2 = b'\x5F\xB0\x45\xA2\x94\x17\xD9\x16\xC6\xC6\xA2\xFF\x06\x41\x82\xB7'
def Encrypt(self, Plaintext : str):
'''
Encrypt plaintext and return corresponding ciphertext.
Args:
Plaintext: A string that will be encrypted.
Returns:
Hexlified ciphertext string.
'''
plain_bytes = Plaintext.encode('utf-16-le')
plain_bytes += b'\x00\x00'
padded_plain_bytes = plain_bytes + os.urandom(Blowfish.block_size - len(plain_bytes) % Blowfish.block_size)
cipher1 = Blowfish.new(self.Key1, Blowfish.MODE_CBC, iv = self.IV)
cipher2 = Blowfish.new(self.Key2, Blowfish.MODE_CBC, iv = self.IV)
return cipher1.encrypt(os.urandom(4) + cipher2.encrypt(padded_plain_bytes) + os.urandom(4)).hex()
def Decrypt(self, Ciphertext : str):
'''
Decrypt ciphertext and return corresponding plaintext.
Args:
Ciphertext: A hex string that will be decrypted.
Returns:
Plaintext string.
'''
cipher1 = Blowfish.new(self.Key1, Blowfish.MODE_CBC, iv = self.IV)
cipher2 = Blowfish.new(self.Key2, Blowfish.MODE_CBC, iv = self.IV)
ciphered_bytes = bytes.fromhex(Ciphertext)
if len(ciphered_bytes) <= 8:
raise ValueError('Invalid Ciphertext.')
padded_plain_bytes = cipher2.decrypt(cipher1.decrypt(ciphered_bytes)[4:-4])
i = 0
for i in range(0, len(padded_plain_bytes), 2):
if padded_plain_bytes[i] == 0 and padded_plain_bytes[i + 1] == 0:
break
plain_bytes = padded_plain_bytes[0:i]
try:
return plain_bytes.decode('utf-16-le')
except UnicodeDecodeError:
raise(ValueError('Invalid Ciphertext.'))
class SecureCRTCryptoV2:
def __init__(self, ConfigPassphrase : str = ''):
'''
Initialize SecureCRTCryptoV2 object.
Args:
ConfigPassphrase: The config passphrase that SecureCRT uses. Leave it empty if config passphrase is not set.
'''
self.IV = b'\x00' * AES.block_size
self.Key = SHA256.new(ConfigPassphrase.encode('utf-8')).digest()
def Encrypt(self, Plaintext : str):
'''
Encrypt plaintext and return corresponding ciphertext.
Args:
Plaintext: A string that will be encrypted.
Returns:
Hexlified ciphertext string.
'''
plain_bytes = Plaintext.encode('utf-8')
if len(plain_bytes) > 0xffffffff:
raise OverflowError('Plaintext is too long.')
plain_bytes = \
len(plain_bytes).to_bytes(4, 'little') + \
plain_bytes + \
SHA256.new(plain_bytes).digest()
padded_plain_bytes = \
plain_bytes + \
os.urandom(AES.block_size - len(plain_bytes) % AES.block_size)
cipher = AES.new(self.Key, AES.MODE_CBC, iv = self.IV)
return cipher.encrypt(padded_plain_bytes).hex()
def Decrypt(self, Ciphertext : str):
'''
Decrypt ciphertext and return corresponding plaintext.
Args:
Ciphertext: A hex string that will be decrypted.
Returns:
Plaintext string.
'''
cipher = AES.new(self.Key, AES.MODE_CBC, iv = self.IV)
padded_plain_bytes = cipher.decrypt(bytes.fromhex(Ciphertext))
plain_bytes_length = int.from_bytes(padded_plain_bytes[0:4], 'little')
plain_bytes = padded_plain_bytes[4:4 + plain_bytes_length]
if len(plain_bytes) != plain_bytes_length:
raise ValueError('Invalid Ciphertext.')
plain_bytes_digest = padded_plain_bytes[4 + plain_bytes_length:4 + plain_bytes_length + SHA256.digest_size]
if len(plain_bytes_digest) != SHA256.digest_size:
raise ValueError('Invalid Ciphertext.')
if SHA256.new(plain_bytes).digest() != plain_bytes_digest:
raise ValueError('Invalid Ciphertext.')
return plain_bytes.decode('utf-8')
if __name__ == '__main__':
import sys
def Help():
print('Usage:')
print(' SecureCRTCipher.py <enc|dec> [-v2] [-p ConfigPassphrase] <plaintext|ciphertext>')
print('')
print(' <enc|dec> "enc" for encryption, "dec" for decryption.')
print(' This parameter must be specified.')
print('')
print(' [-v2] Encrypt/Decrypt with "Password V2" algorithm.')
print(' This parameter is optional.')
print('')
print(' [-p ConfigPassphrase] The config passphrase that SecureCRT uses.')
print(' This parameter is optional.')
print('')
print(' <plaintext|ciphertext> Plaintext string or ciphertext string.')
print(' NOTICE: Ciphertext string must be a hex string.')
print(' This parameter must be specified.')
print('')
def EncryptionRoutine(UseV2 : bool, ConfigPassphrase : str, Plaintext : str):
try:
if UseV2:
print(SecureCRTCryptoV2(ConfigPassphrase).Encrypt(Plaintext))
else:
print(SecureCRTCrypto().Encrypt(Plaintext))
return True
except:
print('Error: Failed to encrypt.')
return False
def DecryptionRoutine(UseV2 : bool, ConfigPassphrase : str, Ciphertext : str):
try:
if UseV2:
print(SecureCRTCryptoV2(ConfigPassphrase).Decrypt(Ciphertext))
else:
print(SecureCRTCrypto().Decrypt(Ciphertext))
return True
except:
print('Error: Failed to decrypt.')
return False
def Main(argc : int, argv : list):
if 3 <= argc and argc <= 6:
bUseV2 = False
ConfigPassphrase = ''
if argv[1].lower() == 'enc':
bEncrypt = True
elif argv[1].lower() == 'dec':
bEncrypt = False
else:
Help()
return -1
i = 2
while i < argc - 1:
if argv[i].lower() == '-v2':
bUseV2 = True
i += 1
elif argv[i].lower() == '-p' and i + 1 < argc - 1:
ConfigPassphrase = argv[i + 1]
i += 2
else:
Help()
return -1
if bUseV2 == False and len(ConfigPassphrase) != 0:
print('Error: ConfigPassphrase is not supported if "-v2" is not specified')
return -1
if bEncrypt:
return 0 if EncryptionRoutine(bUseV2, ConfigPassphrase, argv[-1]) else -1
else:
return 0 if DecryptionRoutine(bUseV2, ConfigPassphrase, argv[-1]) else -1
else:
Help()
exit(Main(len(sys.argv), sys.argv))
SecureCRT-decryptpass.py
#!/usr/bin/env python
#
# Decrypt SSHv2 passwords stored in VanDyke SecureCRT session files
# Can be found on Windows in:
# %APPDATA%\VanDyke\Config\Sessions\sessionname.ini
# Tested with version 7.2.6 (build 606) for Windows
# Eloi Vanderbeken - Synacktiv
# Decrypt SSHv2 passwords stored in VanDyke SecureCRT
# C:\>python SecureCRT-decryptpass.py -h
# usage: SecureCRT-decryptpass.py [-h] files [files ...]
#
#Tool to decrypt SSHv2 passwords in VanDyke Secure CRT session files
#
#positional arguments:
# files session file(s)
#
#optional arguments:
# -h, --help show this help message and exit
#
# C:\>python SecureCRT-decryptpass.py C:\Users\user1\AppData\Roaming\VanDyke\Config\Sessions\192.168.0.1.ini
# C:\Users\user1\AppData\Roaming\VanDyke\Config\Sessions\192.168.0.1.ini
# ssh -p 22 user#192.168.0.1 # 123456
from Crypto.Cipher import Blowfish
import argparse
import re
def decrypt(password) :
c1 = Blowfish.new('5F B0 45 A2 94 17 D9 16 C6 C6 A2 FF 06 41 82 B7'.replace(' ','').decode('hex'), Blowfish.MODE_CBC, '\x00'*8)
c2 = Blowfish.new('24 A6 3D DE 5B D3 B3 82 9C 7E 06 F4 08 16 AA 07'.replace(' ','').decode('hex'), Blowfish.MODE_CBC, '\x00'*8)
padded = c1.decrypt(c2.decrypt(password.decode('hex'))[4:-4])
p = ''
while padded[:2] != '\x00\x00' :
p += padded[:2]
padded = padded[2:]
return p.decode('UTF-16')
REGEX_HOSTNAME = re.compile(ur'S:"Hostname"=([^\r\n]*)')
REGEX_PASWORD = re.compile(ur'S:"Password"=u([0-9a-f]+)')
REGEX_PORT = re.compile(ur'D:"\[SSH2\] Port"=([0-9a-f]{8})')
REGEX_USERNAME = re.compile(ur'S:"Username"=([^\r\n]*)')
def hostname(x) :
m = REGEX_HOSTNAME.search(x)
if m :
return m.group(1)
return '???'
def password(x) :
m = REGEX_PASWORD.search(x)
if m :
return decrypt(m.group(1))
return '???'
def port(x) :
m = REGEX_PORT.search(x)
if m :
return '-p %d '%(int(m.group(1), 16))
return ''
def username(x) :
m = REGEX_USERNAME.search(x)
if m :
return m.group(1) + '#'
return ''
parser = argparse.ArgumentParser(description='Tool to decrypt SSHv2 passwords in VanDyke Secure CRT session files')
parser.add_argument('files', type=argparse.FileType('r'), nargs='+',
help='session file(s)')
args = parser.parse_args()
for f in args.files :
c = f.read().replace('\x00', '')
print f.name
print "ssh %s%s%s # %s"%(port(c), username(c), hostname(c), password(c))
I also encountered the same error and eventually found out that the pycryptodome version was not compatible, I installed the latest version and replaced it with 3.8.2 and it worked fine:
pip install pycryptodome==3.8.2
I would like to send a list of float list via nodejs and receive it in python using protobuff's repeated bytes type.
The graph helps to understand the problem:
I tried with this configuration and what I get on the python side is not really what I expect:
tensors=[b'-TWW', b'-TWW', b'-TWW', b'-TWW']
Here is my test in node.
Client :
const PROTO_PATH = __dirname + '/route_guide.proto';
const async = require('async');
const grpc = require('#grpc/grpc-js');
const protoLoader = require('#grpc/proto-loader');
const packageDefinition = protoLoader.loadSync(
PROTO_PATH,
{
keepCase: true,
longs: String,
enums: String,
defaults: true,
oneofs: true
});
const routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide;
const client = new routeguide.RouteGuide('localhost:50051',
grpc.credentials.createInsecure());
function runJoin(callback) {
const call = client.join();
call.on('data', function(receivedMessage) {
console.log('Got message "' + JSON.stringify(receivedMessage));
});
call.on('end', callback);
messageToSend = {
msg: 'parameters_res',
parameters_res: {
parameters: {
tensors: [
new Buffer.from(new Float64Array([45.1]).buffer),
new Buffer.from(new Float64Array([45.1, 84.5, 87.9, 87.1]).buffer),
new Buffer.from(new Float64Array([45.1, 84.5, 87.9, 87.1]).buffer),
new Buffer.from(new Float64Array([45.1, 84.5, 87.9, 87.1]).buffer)
],
tensor_type: 'numpy.ndarray'
}
}
}
console.log(messageToSend);
console.log(messageToSend.parameters_res.parameters.tensors)
call.write(messageToSend);
call.end();
}
function main() {
async.series([
runJoin
]);
}
if (require.main === module) {
main();
}
exports.runJoin = runJoin;
route_guide.proto:
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.routeguide";
option java_outer_classname = "RouteGuideProto";
option objc_class_prefix = "RTG";
package routeguide;
service RouteGuide {
rpc Join(stream ClientMessage) returns (stream ClientMessage) {}
}
message RouteNote {
repeated bytes model = 1;
}
message ClientMessage {
message Disconnect { Reason reason = 1; }
message ParametersRes { Parameters parameters = 1; }
oneof msg {
Disconnect disconnect = 1;
ParametersRes parameters_res = 2;
}
}
message Parameters {
repeated bytes tensors = 1;
string tensor_type = 2;
}
enum Reason {
UNKNOWN = 0;
RECONNECT = 1;
POWER_DISCONNECTED = 2;
WIFI_UNAVAILABLE = 3;
ACK = 4;
}
Server:
const PROTO_PATH = __dirname + '/route_guide.proto';
const grpc = require('#grpc/grpc-js');
const protoLoader = require('#grpc/proto-loader');
const packageDefinition = protoLoader.loadSync(
PROTO_PATH,
{keepCase: true,
longs: String,
enums: String,
defaults: true,
oneofs: true
});
const routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide;
function join(call) {
call.on('data', function(receivedMessage) {
console.log("SERVER RECEIVE:");
console.log(receivedMessage);
console.log(receivedMessage.parameters_res.parameters.tensors)
for (const element of receivedMessage.parameters_res.parameters.tensors) {
console.log(element)
}
call.write(receivedMessage);
});
call.on('end', function() {
call.end();
});
}
function getServer() {
var server = new grpc.Server();
server.addService(routeguide.RouteGuide.service, {
join: join
});
return server;
}
if (require.main === module) {
var routeServer = getServer();
routeServer.bindAsync('0.0.0.0:50051', grpc.ServerCredentials.createInsecure(), () => {
routeServer.start()
});
}
exports.getServer = getServer;
MyStartegy.py:
from logging import WARNING
from typing import Callable, Dict, List, Optional, Tuple, cast
import numpy as np
import flwr as fl
from flwr.common import (
EvaluateIns,
EvaluateRes,
FitIns,
FitRes,
Parameters,
Scalar,
Weights,
)
from flwr.common.logger import log
from flwr.server.client_manager import ClientManager
from flwr.server.client_proxy import ClientProxy
from flwr.server.strategy.aggregate import aggregate, weighted_loss_avg
from flwr.server.strategy import Strategy
from tensorflow import Tensor
DEPRECATION_WARNING = """
DEPRECATION WARNING: deprecated `eval_fn` return format
loss, accuracy
move to
loss, {"accuracy": accuracy}
instead. Note that compatibility with the deprecated return format will be
removed in a future release.
"""
DEPRECATION_WARNING_INITIAL_PARAMETERS = """
DEPRECATION WARNING: deprecated initial parameter type
flwr.common.Weights (i.e., List[np.ndarray])
will be removed in a future update, move to
flwr.common.Parameters
instead. Use
parameters = flwr.common.weights_to_parameters(weights)
to easily transform `Weights` to `Parameters`.
"""
class MyStrategy(Strategy):
"""Configurable FedAvg strategy implementation."""
# pylint: disable=too-many-arguments,too-many-instance-attributes
def __init__(
self,
fraction_fit: float = 0.1,
fraction_eval: float = 0.1,
min_fit_clients: int = 2,
min_eval_clients: int = 2,
min_available_clients: int = 2,
eval_fn: Optional[
Callable[[Weights], Optional[Tuple[float, Dict[str, Scalar]]]]
] = None,
on_fit_config_fn: Optional[Callable[[int], Dict[str, Scalar]]] = None,
on_evaluate_config_fn: Optional[Callable[[int], Dict[str, Scalar]]] = None,
accept_failures: bool = True,
initial_parameters: Optional[Parameters] = None,
) -> None:
"""Federated Averaging strategy.
Implementation based on https://arxiv.org/abs/1602.05629
Args:
fraction_fit (float, optional): Fraction of clients used during
training. Defaults to 0.1.
fraction_eval (float, optional): Fraction of clients used during
validation. Defaults to 0.1.
min_fit_clients (int, optional): Minimum number of clients used
during training. Defaults to 2.
min_eval_clients (int, optional): Minimum number of clients used
during validation. Defaults to 2.
min_available_clients (int, optional): Minimum number of total
clients in the system. Defaults to 2.
eval_fn (Callable[[Weights], Optional[Tuple[float, float]]], optional):
Function used for validation. Defaults to None.
on_fit_config_fn (Callable[[int], Dict[str, Scalar]], optional):
Function used to configure training. Defaults to None.
on_evaluate_config_fn (Callable[[int], Dict[str, Scalar]], optional):
Function used to configure validation. Defaults to None.
accept_failures (bool, optional): Whether or not accept rounds
containing failures. Defaults to True.
initial_parameters (Parameters, optional): Initial global model parameters.
"""
super().__init__()
self.min_fit_clients = min_fit_clients
self.min_eval_clients = min_eval_clients
self.fraction_fit = fraction_fit
self.fraction_eval = fraction_eval
self.min_available_clients = min_available_clients
self.eval_fn = eval_fn
self.on_fit_config_fn = on_fit_config_fn
self.on_evaluate_config_fn = on_evaluate_config_fn
self.accept_failures = accept_failures
self.initial_parameters = initial_parameters
def __repr__(self) -> str:
rep = f"FedAvg(accept_failures={self.accept_failures})"
return rep
def num_fit_clients(self, num_available_clients: int) -> Tuple[int, int]:
"""Return the sample size and the required number of available
clients."""
num_clients = int(num_available_clients * self.fraction_fit)
return max(num_clients, self.min_fit_clients), self.min_available_clients
def num_evaluation_clients(self, num_available_clients: int) -> Tuple[int, int]:
"""Use a fraction of available clients for evaluation."""
num_clients = int(num_available_clients * self.fraction_eval)
return max(num_clients, self.min_eval_clients), self.min_available_clients
def initialize_parameters(
self, client_manager: ClientManager
) -> Optional[Parameters]:
"""Initialize global model parameters."""
initial_parameters = self.initial_parameters
self.initial_parameters = None # Don't keep initial parameters in memory
if isinstance(initial_parameters, list):
log(WARNING, DEPRECATION_WARNING_INITIAL_PARAMETERS)
initial_parameters = self.weights_to_parameters(weights=initial_parameters)
return initial_parameters
def evaluate(
self, parameters: Parameters
) -> Optional[Tuple[float, Dict[str, Scalar]]]:
"""Evaluate model parameters using an evaluation function."""
if self.eval_fn is None:
# No evaluation function provided
return None
weights = self.parameters_to_weights(parameters)
eval_res = self.eval_fn(weights)
if eval_res is None:
return None
loss, other = eval_res
if isinstance(other, float):
print(DEPRECATION_WARNING)
metrics = {"accuracy": other}
else:
metrics = other
return loss, metrics
def configure_fit(
self, rnd: int, parameters: Parameters, client_manager: ClientManager
) -> List[Tuple[ClientProxy, FitIns]]:
"""Configure the next round of training."""
config = {}
if self.on_fit_config_fn is not None:
# Custom fit config function provided
config = self.on_fit_config_fn(rnd)
fit_ins = FitIns(parameters, config)
# Sample clients
sample_size, min_num_clients = self.num_fit_clients(
client_manager.num_available()
)
clients = client_manager.sample(
num_clients=sample_size, min_num_clients=min_num_clients
)
# Return client/config pairs
return [(client, fit_ins) for client in clients]
def configure_evaluate(
self, rnd: int, parameters: Parameters, client_manager: ClientManager
) -> List[Tuple[ClientProxy, EvaluateIns]]:
"""Configure the next round of evaluation."""
# Do not configure federated evaluation if fraction_eval is 0
if self.fraction_eval == 0.0:
return []
# Parameters and config
config = {}
if self.on_evaluate_config_fn is not None:
# Custom evaluation config function provided
config = self.on_evaluate_config_fn(rnd)
evaluate_ins = EvaluateIns(parameters, config)
# Sample clients
if rnd >= 0:
sample_size, min_num_clients = self.num_evaluation_clients(
client_manager.num_available()
)
clients = client_manager.sample(
num_clients=sample_size, min_num_clients=min_num_clients
)
else:
clients = list(client_manager.all().values())
# Return client/config pairs
return [(client, evaluate_ins) for client in clients]
def aggregate_fit(
self,
rnd: int,
results: List[Tuple[ClientProxy, FitRes]],
failures: List[BaseException],
) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:
"""Aggregate fit results using weighted average."""
if not results:
return None, {}
# Do not aggregate if there are failures and failures are not accepted
if not self.accept_failures and failures:
return None, {}
# Convert results
print("\n\n aggregate_fit")
print(results)
weights_results = [
(self.parameters_to_weights(fit_res.parameters), fit_res.num_examples)
for client, fit_res in results
]
print("weights_results")
print(weights_results)
return self.weights_to_parameters(aggregate(weights_results)), {}
def aggregate_evaluate(
self,
rnd: int,
results: List[Tuple[ClientProxy, EvaluateRes]],
failures: List[BaseException],
) -> Tuple[Optional[float], Dict[str, Scalar]]:
"""Aggregate evaluation losses using weighted average."""
if not results:
return None, {}
# Do not aggregate if there are failures and failures are not accepted
if not self.accept_failures and failures:
return None, {}
loss_aggregated = weighted_loss_avg(
[
(evaluate_res.num_examples, evaluate_res.loss)
for _, evaluate_res in results
]
)
return loss_aggregated, {}
def weights_to_parameters(self, weights: Weights) -> Parameters:
"""Convert NumPy weights to parameters object."""
print('weights_to_parameters')
print(weights)
tensors = [self.ndarray_to_bytes(ndarray) for ndarray in weights]
return Parameters(tensors=tensors, tensor_type="numpy.nda")
def parameters_to_weights(self, parameters: Parameters) -> Weights:
"""Convert parameters object to NumPy weights."""
print('parameters_to_weights')
print(parameters)
return [self.bytes_to_ndarray(tensor) for tensor in parameters.tensors]
# pylint: disable=R0201
def ndarray_to_bytes(self, ndarray: np.ndarray) -> bytes:
"""Serialize NumPy array to bytes."""
print('ndarray_to_bytes')
print(ndarray)
return None
# pylint: disable=R0201
def bytes_to_ndarray(self, tensor: bytes) -> np.ndarray:
"""Deserialize NumPy array from bytes."""
print('bytes_to_ndarray')
print(tensor)
return None
# Start Flower server for three rounds of federated learning
fl.server.start_server(
server_address='localhost:5006',
config={"num_rounds": 2},
strategy=MyStrategy()
)
Is Float64Array the right type?
What should I use on the python side to deserialize the data?
I specify that I cannot modify the proto.
Thank you in advance for your explanations.
I just started studying Lua.
Every time I request, I want to check the name parameter in the request parameter. However, it is actually found that the self.name has changed occasionally.
For example,
request A with params: request_id = 123 & name = ABC,
request B with params: request_id = 321 & name = EFG,
in the log, I found that there are requests_id = 123, but name = EFG.
Why is that? Is my class incorrectly written?
Here is the sample code：
main.lua:
local checker = require "checker"
local ch = checker:new()
if ch:check_name() then
ngx.header["Content-Type"] = "application/json"
ngx.status = ngx.HTTP_FORBIDDEN
ngx.exit(ngx.HTTP_FORBIDDEN)
end
checker.lua:
local utils = require "utils"
local _M = {}
function _M:new()
local o = {}
setmetatable(o, self)
self.__index = self
self.args = utils.get_req_args() or {}
local name = self.args["name"] or ""
local request_id = self.args["request_id"] or ""
self.name = name
return o
end
function _M:check_name()
ngx.log(ngx.ERR, "request_id: ", self.request_id, " name: ", self.name)
-- do some check ...
end
utils.lua
local json = require "cjson"
local _M = {}
function _M.new(self)
return self
end
function _M.get_req_args()
-- GET
local args = nil
if ngx.var.request_method == "GET" then
args = ngx.req.get_uri_args()
-- POST
elseif ngx.var.request_method == "POST" then
ngx.req.read_body()
local data = ngx.req.get_body_data()
args = json.decode(data)
end
return args
end
I have been trying to download all the YouTube comments on popular videos using python requests, but it has been throwing up the following error after about a quarter of the total comments:
{'error': {'code': 400, 'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'errors': [{'message': "The API server failed to successfully process the request. While this can be a transient error, it usually indicates that the request's input is invalid. Check the structure of the commentThread resource in the request body to ensure that it is valid.", 'domain': 'youtube.commentThread', 'reason': 'processingFailure', 'location': 'body', 'locationType': 'other'}]}}
I found this thread detailing the same issue, and it seems that it is not possible to download all the comments on popular videos.
This is my code:
import argparse
import urllib
import requests
import json
import time
start_time = time.time()
class YouTubeApi():
YOUTUBE_COMMENTS_URL = 'https://www.googleapis.com/youtube/v3/commentThreads'
comment_counter = 0
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
def format_comments(self, results, likes_required):
comments_list = []
try:
for item in results["items"]:
comment = item["snippet"]["topLevelComment"]
likes = comment["snippet"]["likeCount"]
if likes < likes_required:
continue
author = comment["snippet"]["authorDisplayName"]
text = comment["snippet"]["textDisplay"]
str = "Comment by {}:\n \"{}\"\n\n".format(author, text)
str = str.encode('ascii', 'replace').decode()
comments_list.append(str)
self.comment_counter += 1
print("Comments downloaded:", self.comment_counter, end="\r")
except(KeyError):
print(results)
return comments_list
def get_video_comments(self, video_id, likes_required):
with open("API_keys.txt", "r") as f:
key_list = f.readlines()
key_list = [key.strip('/n') for key in key_list]
if self.comment_counter <= 900000:
key = self.key_list[0]
elif self.comment_counter <= 1800000:
key = self.key_list[1]
elif self.comment_counter <= 2700000:
key = self.key_list[2]
elif self.comment_counter <= 3600000:
key = self.key_list[3]
elif self.comment_counter <= 4500000:
key = self.key_list[4]
params = {
'part': 'snippet,replies',
'maxResults': 100,
'videoId': video_id,
'textFormat': 'plainText',
'key': key
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
try:
#data = self.openURL(self.YOUTUBE_COMMENTS_URL, params)
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list = []
commments_list += self.format_comments(results, likes_required)
while nextPageToken:
params.update({'pageToken': nextPageToken})
try:
comments_data = requests.get(self.YOUTUBE_COMMENTS_URL, params=params, headers=headers)
except ChunkedEncodingError as c:
tries = 5
print("Chunked Error. Retrying...")
for n in range(tries):
try:
x = 0
x += 1
print("Trying", x, "times")
response = session.post("https://www.youtube.com/comment_service_ajax", params=params, data=data, headers=headers)
comments_data = json.loads(response.text)
except ChunkedEncodingError as c:
print(c)
results = comments_data.json()
nextPageToken = results.get("nextPageToken")
commments_list += self.format_comments(results, likes_required)
return commments_list
def get_video_id_list(self, filename):
try:
with open(filename, 'r') as file:
URL_list = file.readlines()
except FileNotFoundError:
exit("File \"" + filename + "\" not found")
list = []
for url in URL_list:
if url == "\n": # ignore empty lines
continue
if url[-1] == '\n': # delete '\n' at the end of line
url = url[:-1]
if url.find('='): # get id
id = url[url.find('=') + 1:]
list.append(id)
else:
print("Wrong URL")
return list
def main():
yt = YouTubeApi()
parser = argparse.ArgumentParser(add_help=False, description=("Download youtube comments from many videos into txt file"))
required = parser.add_argument_group("required arguments")
optional = parser.add_argument_group("optional arguments")
here: https://console.developers.google.com/apis/credentials")
optional.add_argument("--likes", '-l', help="The amount of likes a comment needs to be saved", type=int)
optional.add_argument("--input", '-i', help="URL list file name")
optional.add_argument("--output", '-o', help="Output file name")
optional.add_argument("--help", '-h', help="Help", action='help')
args = parser.parse_args()
# --------------------------------------------------------------------- #
likes = 0
if args.likes:
likes = args.likes
input_file = "URL_list.txt"
if args.input:
input_file = args.input
output_file = "Comments.txt"
if args.output:
output_file = args.output
list = yt.get_video_id_list(input_file)
if not list:
exit("No URLs in input file")
try:
vid_counter = 0
with open(output_file, "a") as f:
for video_id in list:
vid_counter += 1
print("Downloading comments for video ", vid_counter, ", id: ", video_id, sep='')
comments = yt.get_video_comments(video_id, likes)
if comments:
for comment in comments:
f.write(comment)
print('\nDone!')
except KeyboardInterrupt:
exit("User Aborted the Operation")
# --------------------------------------------------------------------- #
if __name__ == '__main__':
main()
The next best method would be to randomly sample them. Does anyone know if this is possible with the API V3?
Even if the API returns a processingFailure error, you could still catch that (or any other API error for that matter) for to terminate gracefully your pagination loop. This way your script will provide the top-level comments that it fetched from of the API prior to the occurrence of the first API error.
The error response provided by the YouTube Data API is (usually) of the following form:
{
"error": {
"errors": [
{
"domain": <string>,
"reason": <string>,
"message": <string>,
"locationType": <string>,
"location": <string>
}
],
"code": <integer>,
"message": <string>
}
}
Hence, you could have defined the following function:
def is_error_response(response):
error = response.get('error')
if error is None:
return False
print("API Error: "
f"code={error['code']} "
f"domain={error['errors'][0]['domain']} "
f"reason={error['errors'][0]['reason']} "
f"message={error['errors'][0]['message']!r}")
return True
that you'll invoke after each statement of form results = comments_data.json(). In case of the first occurrence of that statement, you'll have:
results = comments_data.json()
if is_error_response(results):
return []
nextPageToken = results.get("nextPageToken")
For the second instance of that statement:
results = comments_data.json()
if is_error_response(results):
return comments_list
nextPageToken = results.get("nextPageToken")
Notice that the function is_error_response above prints out an error message on stdout in case its argument in an API error response; this is for the purpose of having the user of your script informed about the API call failure.
I did Server-Client Application with lua using one Esp8266. I wanna do this with two Esp8266. I wanna use one of these Esp8266 is Server and the other other one is Client. You can see below first code using for get RSSI from one AP and second code is using for writing these RSSI in a Server. How can i placed these two codes in two Esp8266?
i=5
tmr.alarm(1,10000,1, function()
print(wifi.sta.getap(scan_cfg, 1, listap))
if i>1 then
print(i)
i=i-1
else
tmr.stop(1)
print("Timer Durdu")
end
end
)
function listap(t)
for bssid,v in pairs(t) do
local ssid = string.match(v, "([^,]+)")
l=string.format("%-10s",ssid)
stringtoarray = {}
index = 1
for value in string.gmatch(v,"%w+") do
stringtoarray [index] = value
index = index + 1
end
print(l)
print(stringtoarray[2])
end
end
scan_cfg = {}
scan_cfg.ssid = "VSP250s"
scan_cfg.bssid = "00:09:df:8e:03:b4"
scan_cfg.channel = 0
scan_cfg.show_hidden = 1
Second code:
srv=net.createServer(net.TCP)
srv:listen(80,function(conn)
conn:on("receive", function(client,request)
local buf = "";
local _, _, method, path, vars = string.find(request, "([A-Z]+) (.+)?(.+) HTTP");
if(method == nil)then
_, _, method, path = string.find(request, "([A-Z]+) (.+) HTTP");
end
local _GET = {}
if (vars ~= nil)then
for k, v in string.gmatch(vars, "(%w+)=(%w+)&*") do
_GET[k] = v
end
end
buf = buf.."<!DOCTYPE html><html><div id='container'><font size='5'>"
buf = buf..'<style>body{width:auto;height:auto;background-color:#ffffff;}'
buf = buf..'.button {font-size: 20px;}</style>'
buf = buf.."<head> <meta http-equiv='refresh' content=3> "
buf = buf.."<p><h1>RSSI meter<br> ESP8266</h1>";
--buf = buf.."<p>Refresh : <button class='button'>ON</button> </p>";
--buf = buf.."<p>Relay Switch : <button class='button'>ON</button> "
--buf = buf.."<button class='button'>OFF</button><br>"
buf = buf..'<B>Voltage :<font color=red>'..string.format('%s',l)..' V</font></b><br>'
buf = buf..'<B>Current :<B><font color=blue>'..string.format('%g',stringtoarray[2])..' A</font></b><br>'
--buf = buf..'<B>Power Consumption :<B><font color=DeepSkyBlue>'..'Not Available'..'</font></b><br><BR>'
-- buf = buf..'<p>Function Button :<B><font color=BlueViolet>'..button_status..'</font></b><br></p>';
buf = buf..'</head>'
buf = buf..'<br><br><details><summary><font color=red>BURAK IPEK</font><p>'
buf = buf..'<summary><p>Vestel Electronics </p></details>'
buf = buf.."</body></font></div></html>"
client:send(buf);
client:close();
collectgarbage();
end)
end)
Put each code into a lua file. Include both from init.lua with typing
dofile("client.lua");
dofile("server.lua");
To make things easier, write methods.
Good luck.