I have 2 workers
worker(Mmoserver.MessageReceiver, []),
worker(Mmoserver.Main, [])
The MessageReceiver will wait until messages are received on TCP and process them, the Main loop will take that information and act on it. How do I share the info obtained by worker1 with worker2?
Mmoserver.ex
This is the main file that starts the workers
defmodule Mmoserver do
use Application
def start(_type, _args) do
import Supervisor.Spec, warn: false
IO.puts "Listening for packets..."
children = [
# We will add our children here later
worker(Mmoserver.MessageReceiver, []),
worker(Mmoserver.Main, [])
]
# Start the main supervisor, and restart failed children individually
opts = [strategy: :one_for_one, name: AcmeUdpLogger.Supervisor]
Supervisor.start_link(children, opts)
end
end
MessageReceiver.ex
This will just start a tcp listener. It should be able to get a message, figure out what it is (by it's id) then parse data and send it to a specific function in Main
defmodule Mmoserver.MessageReceiver do
use GenServer
require Logger
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, :ok, opts)
end
def init (:ok) do
{:ok, _socket} = :gen_udp.open(21337)
end
# Handle UDP data
def handle_info({:udp, _socket, _ip, _port, data}, state) do
parse_packet(data)
# Logger.info "Received a secret message! " <> inspect(message)
{:noreply, state}
end
# Ignore everything else
def handle_info({_, _socket}, state) do
{:noreply, state}
end
def parse_packet(data) do
# Convert data to string, then split all data
# WARNING - SPLIT MAY BE EXPENSIVE
dataString = Kernel.inspect(data)
vars = String.split(dataString, ",")
# Get variables
packetID = Enum.at(vars, 0)
x = Enum.at(vars, 1)
# Do stuff with them
IO.puts "Packet ID:"
IO.puts packetID
IO.puts x
# send data to main
Mmoserver.Main.handle_data(vars)
end
end
Main.ex
This is the main loop. It will process all the most recent data received by the tcp listener and act on it. Eventually it will update the game state too.
defmodule Mmoserver.Main do
use GenServer
#tickDelay 33
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, [], name: Main)
end
def init (state) do
IO.puts "Main Server Loop started..."
# start the main loop, parameter is the initial tick value
mainLoop(0)
# return, why 1??
{:ok, 1}
end
def handle_data(data) do
GenServer.cast(:main, {:handle_data, data})
end
def handle_info({:handle_data, data}, state) do
# my_function(data)
IO.puts "Got here2"
IO.puts inspect(data)
{:noreply, state}
end
# calls respective game functions
def mainLoop(-1) do
IO.inspect "Server Loop has ended!" # base case, end of loop
end
def mainLoop(times) do
# do shit
# IO.inspect(times) # operation, or body of for loop
# sleep
:timer.sleep(#tickDelay);
# continue the loop RECURSIVELY
mainLoop(times + 1)
end
end
Because Mmoserver.MessageReceiver is going to send messages to Mmoserver.Main, Main has to be started in first place, plus, it needs to have name associated:
worker(Mmoserver.Main, []),
worker(Mmoserver.MessageReceiver, [])
The easiest way could be, in your Mmoserver.Main, assuming it is a GenServer:
defmodule Mmoserver.Main do
use GenServer
def start_link do
GenServer.start_link(__MODULE__, [], name: :main)
end
# ...
end
You can add convenience function, plus the implementation one like:
defmodule Mmoserver.Main do
# ...
def handle_data(data) do
GenServer.cast(:main, {:handle_data, data})
end
def handle_info({:handle_data, data}, state) do
my_function(data)
{:noreply, state}
end
end
So, your MessageReceiver, can send a message like:
defmodule Mmoserver.MessageReceiver do
def when_data_received(data) do
Mmoserver.Main.handle_data(data)
end
end
This assumes Mmoserver.MessageReceiver doesn't expect Mmoserver.Main to respond. I've decided to do it this way as you didn't specify the way you want to handle the data and this seems the easies example of how to do this.
Related
I have a large legacy application that has one function that is a prime candidate to be executed async. It's IO bound (network and disk) and doesn't return anything.
This is a very simple similar implementation:
import random
import time
import requests
def fetch_urls(site):
wait = random.randint(0, 5)
filename = site.split("/")[2].replace(".", "_")
print(f"Will fetch {site} in {wait} seconds")
time.sleep(wait)
r = requests.get(site)
with open(filename, "w") as fd:
fd.write(r.text)
def something(sites):
for site in sites:
fetch_urls(site)
return True
def main():
sites = ["https://www.google.com", "https://www.reddit.com", "https://www.msn.com"]
start = time.perf_counter()
something(sites)
total_time = time.perf_counter() - start
print(f"Finished in {total_time}")
if __name__ == "__main__":
main()
My end goal would be updating the something function to run fetch_urls async.
I cannot change fetch_urls.
All documentation and tutorials I can find assumes my entire application is async (starting from async def main()) but this is not the case.
It's a huge application spanning across multiple modules and re-factoring everything for a single function doesn't look right.
For what I understand I will need to create a loop, add tasks to it and dispatch it somehow, but I tried many different things and I still get everything running just one after another - as oppose to concurrently.
I would appreciate any assistance. Thanks!
Replying to myself, it seems there is no easy way to do that with async. Ended up using concurrent.futures
import time
import requests
import concurrent.futures
def fetch_urls(url, name):
wait = 5
filename = url.split("/")[2].replace(".", "_")
print(f"Will fetch {name} in {wait} seconds")
time.sleep(wait)
r = requests.get(url)
with open(filename, "w") as fd:
fd.write(r.text)
def something(sites):
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
future_to_url = {
executor.submit(fetch_urls, url["url"], url["name"]): (url)
for url in sites["children"]
}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try:
data = future.result()
except Exception as exc:
print("%r generated an exception: %s" % (url, exc))
return True
def main():
sites = {
"parent": "https://stackoverflow.com",
"children": [
{"name": "google", "url": "https://google.com"},
{"name": "reddit", "url": "https://reddit.com"},
],
}
start = time.perf_counter()
something(sites)
total_time = time.perf_counter() - start
print(f"Finished in {total_time}")
This program is an API based program that has been working for a few months and all of a sudden has went days without pushing anything to Discord. The script looks fine in CMD, but no errors are being thrown. I was wondering if there was a way to eliminate possible issues such as an API instability issue or something obvious. The program is supposed to go to the site www.bitskins.com and pull skins based on parameters set and push them as an embed to a Discord channel every 10 minutes.
There are two files that run this program.
Here is the one that uses Bitskins API (bitskins.py):
import requests, json
from datetime import datetime, timedelta
class Item:
def __init__(self, item):
withdrawable_at= item['withdrawable_at']
price= float(item['price'])
self.available_in= withdrawable_at- datetime.timestamp(datetime.now())
if self.available_in< 0:
self.available= True
else:
self.available= False
self.suggested_price= float(item['suggested_price'])
self.price= price
self.margin= round(self.suggested_price- self.price, 2)
self.reduction= round((1- (self.price/self.suggested_price))*100, 2)
self.image= item['image']
self.name= item['market_hash_name']
self.item_id= item['item_id']
def __str__(self):
if self.available:
return "Name: {}\nPrice: {}\nSuggested Price: {}\nReduction: {}%\nAvailable Now!\nLink: https://bitskins.com/view_item?app_id=730&item_id={}".format(self.name, self.price, self.suggested_price, self.reduction, self.item_id)
else:
return "Name: {}\nPrice: {}\nSuggested Price: {}\nReduction: {}%\nAvailable in: {}\nLink: https://bitskins.com/view_item?app_id=730&item_id={}".format(self.name, self.price, self.suggested_price, self.reduction, str(timedelta(seconds= self.available_in)), self.item_id)
def __lt__(self, other):
return self.reduction < other.reduction
def __gt__(self, other):
return self.reduction > other.reduction
def get_url(API_KEY, code):
PER_PAGE= 30 # the number of items to retrieve. Either 30 or 480.
return "https://bitskins.com/api/v1/get_inventory_on_sale/?api_key="+ API_KEY+"&code=" + code + "&per_page="+ str(PER_PAGE)
def get_data(url):
r= requests.get(url)
data= r.json()
return data
def get_items(code, API_KEY):
url= get_url(API_KEY, code)
try:
data= get_data(url)
if data['status']=="success":
items= []
items_dic= data['data']['items']
for item in items_dic:
tmp= Item(item)
if tmp.reduction>=25 and tmp.price<=200: # Minimum discount and maximum price to look for when grabbing items. Currently set at minimum discount of 25% and maxmimum price of $200.
items.append(tmp)
return items
else:
raise Exception(data["data"]["error_message"])
except:
raise Exception("Couldn't connect to BitSkins.")
# my_token = pyotp.TOTP(my_secret)
# print(my_token.now()) # in python3
And here is the file with Discord's API (solution.py):
#!/bin/env python3.6
import bitskins
import discord
import pyotp, base64, asyncio
from datetime import timedelta, datetime
TOKEN= "Not input for obvious reasons"
API_KEY= "Not input for obvious reasons"
my_secret= 'Not input for obvious reasons'
client = discord.Client()
def get_embed(item):
embed=discord.Embed(title=item.name, url= "https://bitskins.com/view_item?app_id=730&item_id={}".format(item.item_id), color=0xA3FFE8)
embed.set_author(name="Skin Bot", url="http://www.reactor.gg/",icon_url="https://pbs.twimg.com/profile_images/1050077525471158272/4_R8PsrC_400x400.jpg")
embed.set_thumbnail(url=item.image)
embed.add_field(name="Price:", value="${}".format(item.price))
embed.add_field(name="Discount:", value="{}%".format(item.reduction), inline=True)
if item.available:
tmp= "Instantly Withdrawable"
else:
tmp= str(timedelta(seconds= item.available_in))
embed.add_field(name="Availability:", value=tmp, inline=True)
embed.add_field(name="Suggested Price:", value="${}".format(item.suggested_price), inline=True)
embed.add_field(name="Profit:", value="${}".format(item.margin), inline=True)
embed.set_footer(text="Made by Aqyl#0001 | {}".format(datetime.now()), icon_url="https://www.discordapp.com/assets/6debd47ed13483642cf09e832ed0bc1b.png")
return embed
async def status_task(wait_time= 60* 5):
while True:
print("Updated on: {}".format(datetime.now()))
code= pyotp.TOTP(my_secret)
try:
items= bitskins.get_items(code.now(), API_KEY)
for item in items:
await client.send_message(client.get_channel("656913641832185878"), embed=get_embed(item))
except:
pass
await asyncio.sleep(wait_time)
#client.event
async def on_ready():
wait_time= 60 * 10 # 10 mins in this case
print('CSGO BitSkins Bot')
print('Made by Aqyl#0001')
print('Version 1.0.6')
print('')
print('Logged in as:')
print(client.user.name)
print('------------------------------------------')
client.loop.create_task(status_task(wait_time))
try:
client.run(TOKEN)
except:
print("Couldn't connect to the Discord Server.")
You have a general exception, this will lead to catching exceptions that you really don't want to catch.
try:
items= bitskins.get_items(code.now(), API_KEY)
for item in items:
await client.send_message(client.get_channel("656913641832185878"), embed=get_embed(item))
except:
pass
This is the same as catching any exception that appears there (Exceptions that inherit BaseException
To avoid those problems, you should always catch specific exceptions. (i.e. TypeError).
Example:
try:
raise Exception("Example exc")
except Exception as e:
print(f"Exception caught! {e}")
I must create an api with great performance and I want to create it with Elixir
I have a process (slow) that I must run after some requests. I want to make this flow
In each request, save the data received in memory
After x requests, send to another api (or after x seconds)
In node I can make this:
let batchData = []
const handlerRequest = (req, res) => {
batchData.push(req. body.data)
if (batchData > 1000) {
// Process to send to another api
batchData = []
}
res.json({ success: true })
}
Or
let batchData = []
setInterval(() => {
if (batchData > 1000) {
// Process to send to another api
batchData = []
}
}, 10000)
const handlerRequest = (req, res) => {
batchData.push(req. body.data)
res.json({ success: true })
}
How can I do something like this in Elixir Phoenix?
Thanks for this
Here is an approach using a GenServer. I presume you want to start the timer when the first item is received.
defmodule RequestHandler do
use GenServer
#name __MODULE__
#timeout 5_000
#size 5
def start_link(args \\ []) do
GenServer.start_link(__MODULE__, args, name: #name)
end
def request(req) do
GenServer.cast(#name, {:request, req})
end
def init(_) do
{:ok, %{timer_ref: nil, requests: []}}
end
def handle_cast({:request, req}, state) do
{:noreply, state |> update_in([:requests], & [req | &1]) |> handle_request()}
end
def handle_info(:timeout, state) do
# sent to another API
send_api(state.requests)
{:noreply, reset_requests(state)}
end
defp handle_request(%{requests: requests} = state) when length(requests) == 1 do
start_timer(state)
end
defp handle_request(%{requests: requests} = state) when length(requests) > #size do
# sent to another API
send_api(requests)
reset_requests(state)
end
defp handle_request(state) do
state
end
defp reset_requests(state) do
state
|> Map.put(:requests, [])
|> cancel_timer()
end
defp start_timer(state) do
timer_ref = Process.send_after(self(), :timeout, #timeout)
state
|> cancel_timer()
|> Map.put(:timer_ref, timer_ref)
end
defp cancel_timer(%{timer_ref: nil} = state) do
state
end
defp cancel_timer(%{timer_ref: timer_ref} = state) do
Process.cancel_timer(timer_ref)
Map.put(state, :timer_ref, nil)
end
defp send_api(requests) do
IO.puts "sending #{length requests} requests"
end
end
And here is a few tests
iex(5)> RequestHandler.start_link
{:ok, #PID<0.119.0>}
iex(6)> for i <- 1..6, do: Request
[Request, Request, Request, Request, Request, Request]
iex(7)> for i <- 1..6, do: RequestHandler.request(i)
sending 6 requests
[:ok, :ok, :ok, :ok, :ok, :ok]
iex(8)> for i <- 1..7, do: RequestHandler.request(i)
sending 6 requests
[:ok, :ok, :ok, :ok, :ok, :ok, :ok]
sending 1 requests
iex(9)> for i <- 1..3, do: RequestHandler.request(i)
[:ok, :ok, :ok]
sending 3 requests
iex(10)>
You can use GenServer or Agent
GenServer
The general idea is to have a GenServer process that holds the data to be processed and also handles the background processing. Using GenServer.cast/2 we can send a message to a process asynchronously. So whenever, the controller receives the request, we'll add a new item to the queue and also check if batch size is reached and process it.
# In Controller (page_controller.ex) module
def index(conn, params) do
App.BatchProcessor.add_item(params)
conn|>json(%{success: true})
end
Add module for GenServer. You can add a new file lib/batch_processor.ex
defmodule App.BatchProcessor do
use GenServer
#batch_size 10 #whenever queue reaches this size we'll start processing
def init(_) do
initial_queue = []
{:ok, initial_queue}
end
def start_link()do
GenServer.start_link(__MODULE__, [], [name: __MODULE__])
end
#api function to add item to the
def add_item(data)do
GenServer.cast({:add, data}, __MODULE__)
end
# implement GenServer behavior function to handle cast messages for adding item to the queue
def handle_cast({:add, data}, queue) do
update_queue = [data | queue] #addpend new item to front of queue
#check if batch size is reached and process current batch
if Enum.count(updated_queue) >= #batch_size do
#send async message to current process to process batch
GenServer.cast(__MODULE__, :process_batch)
end
{:noreply, updated_queue}
end
#implement GenServer behavior function to handle cast messages for processing batch
def handle_cast(:process_queue, queue)do
spawn(fn ->
Enum.each(queue, fn data ->
IO.inspect(data)
end)
end)
{:noreply, []} # reset queue to empty
end
end
Start the BatchProcessor process when the Phoenix app starts
#application.ex
children = [
# Start the endpoint when the application starts
supervisor(App.Web.Endpoint, []),
# Start your own worker by calling: App.Web.Worker.start_link(arg1, arg2, arg3)
worker(App.BatchProcessor, []),
]
Read more on GenServer
Hope this helps
Given a running GenServer, is there a known way to send synchronous/asynchronous calls to the pid via an endpoint, without using the Phoenix framework?
Here's an example call (using python's requests library) that maps the reply term to JSON:
iex> give_genserver_endpoint(pid, 'http://mygenserverendpoint/api')
iex> {:ok, 'http://mygenserverendpoint/api'}
>>> requests.get(url='http://mygenserverendpoint/getfood/fruits/colour/red')
>>> '{ "hits" : ["apple", "plum"]}'
You can write a complete elixir http server using cowboy and plug:
Application Module
defmodule MyApp do
use Application
def start(_type, _args) do
import Supervisor.Spec
children = [
worker(MyGenServer, []),
Plug.Adapters.Cowboy.child_spec(:http, MyRouter, [], [port: 4001])
]
opts = [strategy: :one_for_one, name: MyApp.Supervisor]
Supervisor.start_link(children, opts)
end
end
Router Module
defmodule MyRouter do
use Plug.Router
plug :match
plug :dispatch
get "/mygenserverendpoint/getfood/fruits/colour/:colour" do
response_body = MyGenServer.get_fruit_by_colour(colour)
conn
|> put_resp_content_type("application/json")
|> send_resp(conn, 200, Poison.encode(response_body))
end
match _ do
send_resp(conn, 404, "oops")
end
end
GenServer module
defmodule MyGenServer do
use GenServer
def start_link do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end
def get_fruit_by_colour(colour) do
GenServer.call(__MODULE__, {:get_by_colour, colour})
end
def handle_call({:get_by_colour, colour}, _from, state) do
{:reply, %{"hits" : ["apple", "plum"]}, state}
end
end
I am using the following code to create a TCP listener on elixir :
defmodule KVServer do
use Application
#doc false
def start(_type, _args) do
import Supervisor.Spec
children = [
supervisor(Task.Supervisor, [[name: KVServer.TaskSupervisor]]),
worker(Task, [KVServer, :accept, [4040]])
]
opts = [strategy: :one_for_one, name: KVServer.Supervisor]
Supervisor.start_link(children, opts)
end
#doc """
Starts accepting connections on the given `port`.
"""
def accept(port) do
{:ok, socket} = :gen_tcp.listen(port,
[:binary, packet: :line, active: false, reuseaddr: true])
IO.puts "Accepting connections on port #{port}"
loop_acceptor(socket)
end
defp loop_acceptor(socket) do
{:ok, client} = :gen_tcp.accept(socket)
{:ok, pid} = Task.Supervisor.start_child(KVServer.TaskSupervisor, fn -> serve(client) end)
:ok = :gen_tcp.controlling_process(client, pid)
loop_acceptor(socket)
end
defp serve(socket) do
socket
|> read_line()
|> write_line(socket)
serve(socket)
end
defp read_line(socket) do
{:ok, data} = :gen_tcp.recv(socket, 0)
data
end
defp write_line(line, socket) do
:gen_tcp.send(socket, line)
end
end
It is taken from the following link: http://elixir-lang.org/getting-started/mix-otp/task-and-gen-tcp.html
When I try to get data from my gps deveice (for which I am writing this piece of code) using :gen_tcp.recv(socket,0), I get an error:
{:error, reason} = :gen_tcp.recv(socket, 0) and the reason it showed is just "closed".
However, the device is sending data, which I confirmed using a tcp packet sniffer (tcpflow).
Also, when I try to send data using telnet as described in the tutorial above, it works fine.
Any help would be highly appreciated.
I was finally able to figure it out. Actually the device was sending raw stream of data not lines of data. So I had to change "packet: :line" parameter in :gen_tcp.listen function to "packet: :raw".
It was working on telnet because telnet sends lines of data (with line breaks).