Ok so I built a semi-simple multiplayer game that connects through UDP. However I can't get it to fully connect outside my local network. Now! I have seen this question asked and answered many times so here is where it differs from similar questions:
I have already port forwarded the correct ports.
That's right! I am a Minecraft veteran so the one thing I do know how to do is forward ports. ;)
Now, this works fine on my local network. It works if I run two instances on the same machine of course. But it also works if I use a different computer connecting to the LAN IP or if I use a different computer connecting to my routers public address. The connection only seems to fail if the messages are originating from outside the network they are being hosted on.
The host receives incoming messages from clients, but clients do not receive messages from host. And it doesn't mater who is the host and who is the client. So I had my friend try connecting to my game from his house and had a break point on the function that receives the messages. Sure enough I see the expected message come into the host, and then the host sends out the acknowledgement message, but his client never received it. Similarly, if my friend hosts and I try to connect, his host sees the message and sends the acknowledgment but my client never receives it. So at this point we both have the port forwarded but only the host ever receives anything.
So.... Anyone have any idea what I might be doing wrong? Or what Cthulhu of the internet I need to kill?
My Code Below:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
public enum MessageType
{
None = 0,
Connect = 1,
Command = 2,
EntityUpdate = 3,
Response = 4,
}
[System.Serializable]
public class Message
{
public MessageType _MessageType = MessageType.None;
public object Data = null;
public Guid Origin = Guid.Empty;
public Guid MessageID;
public Message(object data, MessageType type, Guid origin)
{
if (type == MessageType.None || data == null)
{
throw new Exception("Invalid Message!");
}
Data = data;
_MessageType = type;
Origin = origin;
MessageID = Guid.NewGuid();
}
}
public static class NetLayer
{
private const int FRAMES_BETWEEN_SENDS = 2;
private const int NUM_RETRIES = 10;
private const int FAMES_BEFORE_RETRY_ENTITY = 120;
private const int FAMES_BEFORE_RETRY_CLIENT = 30;
private const int BUFF_SIZE = 8 * 2048;
private const int DEFAULT_PORT = 27015;
private const string DEFAULT_ADDRESS = "localhost";
private const int MAX_BYTES = 1100;
private const int MAX_BYTES_PER_PACKET = 1100 - (322 + 14); //281 is number of existing bytes in empty serilized packet //14 is... I dunno. the overhead for having the byte aray populated I assume
private static Socket mSocket = null;
private static List<EndPoint> mClients = null;
private static Dictionary<EndPoint, PlayerData> mUnconfirmedClients = null;
private static string mServerAdress = DEFAULT_ADDRESS;
private static int mPort = DEFAULT_PORT;
public static bool IsServer { get { return mClients != null; } }
public static bool IsConnected = false;
public static bool Initiated { get { return mSocket != null; } }
private static Dictionary<Guid, Packet> mUnconfirmedPackets = null;
private static Queue<Packet> ResendQueue = new Queue<Packet>();
private static EndPoint mCurrentPacketEndpoint;
private static Guid mCurrentMessageID;
private static byte[] mDataToSend;
private static int mCurrentDataIndex = 0;
private static int mCurrentPacketIndex = 0;
private static int mTotalPacketsToSend = 0;
private static Queue<Packet> ResponseQueue = new Queue<Packet>();
private static Dictionary<Guid, Dictionary<int, Packet>> IncomingPacketGroups = new Dictionary<Guid, Dictionary<int, Packet>>();
private static Queue<QueueuedMessage> MessageQueue = new Queue<QueueuedMessage>();
public static int QueueCount { get { return MessageQueue.Count; } }
private static int mCurrentFramesToRetryClient;
private static int mCurrentFramesToRetryEntity;
private static int mLastSendFrame = 0;
[System.Serializable]
public class Packet
{
public byte[] Data;
public Guid MessageID;
public Guid PacketID;
public int PacketNumber;
public int NumPacketsInGroup;
public int NumBytesInGroup;
public bool RequireResponse;
[System.NonSerialized]
public int retries; //We don't need to send this data, just record it locally
[System.NonSerialized]
public EndPoint Destination; //We don't need to send this data, just record it locally
public Packet(byte[] data, Guid messageID, int packetNumber, int numPacketsinGroup, int numBytesInGroup, EndPoint destination, bool requireResponse)
{
Data = data;
if(data.Length > MAX_BYTES_PER_PACKET)
{
Debug.LogError("Creating a packet with a data size of " + data.Length + " which is greater then max data size of " + MAX_BYTES_PER_PACKET);
}
MessageID = messageID;
PacketID = Guid.NewGuid();
PacketNumber = packetNumber;
NumPacketsInGroup = numPacketsinGroup;
NumBytesInGroup = numBytesInGroup;
retries = 0;
Destination = destination;
RequireResponse = requireResponse;
}
}
private class QueueuedMessage
{
public Message m;
public EndPoint e;
public QueueuedMessage(Message message, EndPoint endpoint)
{
m = message;
e = endpoint;
}
}
[System.Serializable]
private class PacketResponse
{
public Guid MessageIDToConfirm;
public PacketResponse(Guid newGuid)
{
MessageIDToConfirm = newGuid;
}
}
public static void InitAsServer(string address, int port)
{
Reset();
mClients = new List<EndPoint>();
mUnconfirmedClients = new Dictionary<EndPoint, PlayerData>();
mSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
mSocket.SetSocketOption(SocketOptionLevel.IP, SocketOptionName.ReuseAddress, true);
mSocket.Blocking = false;
mSocket.SendBufferSize = BUFF_SIZE;
mSocket.ReceiveBufferSize = BUFF_SIZE;
mSocket.Bind(new IPEndPoint(IPAddress.Parse(address), port));
mPort = port;
IsConnected = true;
mUnconfirmedPackets = new Dictionary<Guid, Packet>();
Packet newPacket = new Packet(new byte[MAX_BYTES_PER_PACKET], Guid.NewGuid(), 1, 15, MAX_BYTES_PER_PACKET, null, true);
byte[] serilized = Util.Serialize(newPacket);
Debug.Log("Total Packet Size = " + serilized.Length);
}
public static void InitAsClient(string address, int port)
{
Reset();
mSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
mSocket.Blocking = false;
mSocket.SendBufferSize = BUFF_SIZE;
mSocket.ReceiveBufferSize = BUFF_SIZE;
mSocket.Bind(new IPEndPoint(IPAddress.Parse(GetLocalIPAddress()), port));
mServerAdress = address;
mPort = port;
mUnconfirmedPackets = new Dictionary<Guid, Packet>();
}
public static void NetTick()
{
if (Initiated)
{
//Keep retrying connection messages untill we get a response from the player, in case they miss our first message
if(IsServer)
{
mCurrentFramesToRetryClient++;
if (mCurrentFramesToRetryClient >= FAMES_BEFORE_RETRY_CLIENT)
{
mCurrentFramesToRetryClient = 0;
foreach (EndPoint ep in mUnconfirmedClients.Keys)
{
Debug.Log("Pinging client again: " + ep);
SendConnectionConfirmationToClient(ep, mUnconfirmedClients[ep]); //TODO Add timeout. Should stop pinging after 10 attempts
}
}
}
//Retry messages untill we get a responce or give up
mCurrentFramesToRetryEntity++;
if (mCurrentFramesToRetryEntity >= FAMES_BEFORE_RETRY_ENTITY)
{
List<Guid> deadMessages = new List<Guid>();
List<Guid> resendMessages = new List<Guid>();
mCurrentFramesToRetryEntity = 0;
foreach (Guid messageID in mUnconfirmedPackets.Keys)
{
Packet packet = mUnconfirmedPackets[messageID];
packet.retries++;
if (packet.retries > NUM_RETRIES)
{
Debug.LogError("Failed to send Packet " + messageID);
deadMessages.Add(messageID);
}
else
{
resendMessages.Add(messageID);
}
}
//Resend everything we have selected to resend
for (int i = 0; i < resendMessages.Count; i++)
{
Packet packet = mUnconfirmedPackets[resendMessages[i]];
ResendQueue.Enqueue(packet);
}
//Give up on messages that have been marked dead
for (int i = 0; i < deadMessages.Count; i++)
{
mUnconfirmedPackets.Remove(deadMessages[i]);
}
}
if (Time.frameCount - mLastSendFrame >= FRAMES_BETWEEN_SENDS)
{
if (ResponseQueue.Count > 0) //Send responses first
{
Packet p = ResponseQueue.Dequeue();
SendImmediate(p, p.Destination, false);
}
else if (ResendQueue.Count > 0) //Then resends
{
Packet p = ResendQueue.Dequeue();
SendImmediate(p, p.Destination, false);
}
else //Then normal messages
{
StreamSend();
}
mLastSendFrame = Time.frameCount;
}
}
}
public static string GetLocalIPAddress()
{
var host = Dns.GetHostEntry(Dns.GetHostName());
foreach (var ip in host.AddressList)
{
if (ip.AddressFamily == AddressFamily.InterNetwork)
{
return ip.ToString();
}
}
return null;
}
public static void Reset()
{
if(mSocket != null)
{
mSocket.Close();
}
mSocket = null;
mClients = null;
mServerAdress = DEFAULT_ADDRESS;
mPort = DEFAULT_PORT;
IsConnected = false;
}
public static void Send(Message newMessage, EndPoint endpoint = null)
{
if (IsServer)
{
if (endpoint != null)
{
SendToQueue(new QueueuedMessage(newMessage, endpoint));
}
else
{
for (int i = 0; i < mClients.Count; i++)
{
SendToQueue(new QueueuedMessage(newMessage, mClients[i]));
}
}
}
else
{
SendToQueue(new QueueuedMessage(newMessage, new IPEndPoint(IPAddress.Parse(mServerAdress), mPort)));
}
}
private static void SendToQueue(QueueuedMessage newPacket)
{
MessageQueue.Enqueue(newPacket);
}
private static void StreamSend()
{
if(mDataToSend != null)
{
//Get number of bytes to send in this packet
int bytesToSend = Mathf.Min(mDataToSend.Length - mCurrentDataIndex, MAX_BYTES_PER_PACKET);
//Populate packet byte array
byte[] data = new byte[bytesToSend];
for(int i = 0; i < bytesToSend; i++)
{
data[i] = mDataToSend[mCurrentDataIndex + i];
}
//Increment index by the nubmer of bytes sent
mCurrentDataIndex += bytesToSend;
//Create and send packet
Packet newPacket = new Packet(data, mCurrentMessageID, mCurrentPacketIndex, mTotalPacketsToSend, mDataToSend.Length, mCurrentPacketEndpoint, true);
SendImmediate(newPacket, mCurrentPacketEndpoint, true);
//Increment packet index
mCurrentPacketIndex++;
if(mCurrentDataIndex >= mDataToSend.Length)
{
//We have finished sending. Clear data array
mDataToSend = null;
}
}
else if(MessageQueue.Count > 0)
{
//Prepare the next group for packing
QueueuedMessage queueuedMessage = MessageQueue.Dequeue();
mCurrentMessageID = queueuedMessage.m.MessageID;
mCurrentPacketEndpoint = queueuedMessage.e;
mDataToSend = Util.Serialize(queueuedMessage.m);
mCurrentDataIndex = 0;
mCurrentPacketIndex = 1;
mTotalPacketsToSend = Mathf.CeilToInt((float)mDataToSend.Length / (float)MAX_BYTES_PER_PACKET);
}
}
public static void SendImmediate(Packet packet, EndPoint endpoint, bool verify)
{
byte[] data = Util.Serialize(packet);
if(data.Length > MAX_BYTES)
{
Debug.LogError("Attempting to send packet with " + data.Length + " bytes. This will most likely fail! Please ensure all packets are limited to " + MAX_BYTES + " bytes or less!");
}
mSocket.SendTo(data, endpoint);
//Don't assume the packet is late untill we have actually sent it.
//The old way of verifying. Instead things sending imediate that don't want to be veiried should set verify to false if (newMessage._MessageType != MessageType.Response && newMessage._MessageType != MessageType.Connect && !mUnconfirmedPackets.ContainsKey(newMessage.MessageID))
if (verify)
{
mUnconfirmedPackets.Add(packet.PacketID, packet);
}
}
public static void SendConnectionMessage(PlayerData playerData)
{
Message message = new Message(playerData, MessageType.Connect, playerData.ID);
Send(message);
}
public static void SendCommand(Command command, EndPoint ep = null)
{
Message message = new Message(command, MessageType.Command, GameManager.LocalPlayer.PlayerID);
Send(message, ep);
}
public static void SendEntityUpdate(EntityData entityData, EndPoint ep = null)
{
Message message = new Message(entityData, MessageType.EntityUpdate, GameManager.LocalPlayer.PlayerID);
Send(message, ep);
}
public static void Receive()
{
if (Initiated && mSocket.Available > 0)
{
byte[] bytes = new byte[BUFF_SIZE];
EndPoint epFrom = new IPEndPoint(IPAddress.Any, 0); //TODO Do something here to stop reciving messages from dead clients...
mSocket.ReceiveFrom(bytes, ref epFrom);
Reassemble(bytes, epFrom);
}
}
public static void Reassemble(byte[] bytes, EndPoint epFrom)
{
Packet receivedPacket = Util.Deserilize(bytes) as Packet;
//Create new packet group if one does not already exist
if(!IncomingPacketGroups.ContainsKey(receivedPacket.MessageID))
{
IncomingPacketGroups.Add(receivedPacket.MessageID, new Dictionary<int, Packet>());
}
Dictionary<int, Packet> currentPacketGroup = IncomingPacketGroups[receivedPacket.MessageID];
currentPacketGroup[receivedPacket.PacketNumber] = receivedPacket;
if(currentPacketGroup.Count == receivedPacket.NumPacketsInGroup)
{
byte[] completedData = new byte[receivedPacket.NumBytesInGroup];
int index = 0;
for(int i = 0; i < receivedPacket.NumPacketsInGroup; i++)
{
Packet currentPacket = currentPacketGroup[i + 1];
for(int j = 0; j < currentPacket.Data.Length; j++)
{
completedData[index] = currentPacket.Data[j];
index++;
}
}
//Clean up the packet group now that it is compleated
IncomingPacketGroups.Remove(receivedPacket.MessageID);
//Call this once we have re-assembled a whole message
Message receivedMessage = Util.Deserilize(completedData) as Message;
ReceiveCompleateMessage(receivedMessage, epFrom);
}
//Make sure to call back to sender to let them know we got this packet
if(receivedPacket.RequireResponse)
{
SendMessageResponse(receivedPacket.PacketID, epFrom);
}
}
private static void ReceiveCompleateMessage(Message receivedMessage, EndPoint epFrom)
{
if (receivedMessage.Origin == GameManager.LocalPlayer.PlayerID)
{
//Reject messages we sent ourselves. Prevent return from sender bug.
return;
}
//TODO Consider adding a check that adds the endpoing into the connections even if we don't have a connecton message because players can sometimes re-connect.
//TODO or maybe reject messages from unknown clients becasue we shouldn't trust them
switch (receivedMessage._MessageType)
{
case MessageType.None:
Debug.LogError("Recived message type of None!");
break;
case MessageType.EntityUpdate:
GameManager.ProcessEntityUpdate(receivedMessage);
break;
case MessageType.Command:
GameManager.ProcessCommand(receivedMessage);
break;
case MessageType.Response:
ProcessResponse(receivedMessage);
break;
case MessageType.Connect:
ProcessConnection(receivedMessage, epFrom);
break;
}
//If this is the server reciving a messgae, and that message was not sent by us
if (IsServer && receivedMessage._MessageType != MessageType.Connect && receivedMessage._MessageType != MessageType.Response && receivedMessage.Origin != GameManager.LocalPlayer.PlayerID)
{
//Mirror the message to all connected clients
Send(receivedMessage);
}
}
public static void ProcessConnection(Message receivedMessage, EndPoint epFrom)
{
//This is the only time where we should process the data directly in net layer since we need the endpoint data
if (IsServer)
{
if (!mClients.Contains(epFrom))
{
if (epFrom != null)
{
mClients.Add(epFrom);
Debug.Log("Client attempting to join: " + epFrom.ToString()); //TODO becasue we are binding socket this won't work locally anymore but see if it at least works globally before trying to fix
//Get new player from recived data
PlayerData newPlayer = receivedMessage.Data as PlayerData;
newPlayer.Team = GameManager.GetNewTeam();
newPlayer.HostHasResponded = true;
if (!mUnconfirmedClients.ContainsKey(epFrom))
{
mUnconfirmedClients.Add(epFrom, newPlayer);
}
SendConnectionConfirmationToClient(epFrom, newPlayer);
}
else
{
Debug.LogError("Got connection message from null!");
}
}
else
{
if (mUnconfirmedClients.ContainsKey(epFrom))
{
PlayerData newPlayerData = receivedMessage.Data as PlayerData;
if (newPlayerData.HostHasResponded)
{
mUnconfirmedClients.Remove(epFrom);
if (epFrom != null)
{
Debug.Log("Client Confirmed, sending update: " + epFrom.ToString());
//Sent all known entitys to the new client so it's up to date
foreach (Guid key in GameManager.Entities.Keys)
{
SendEntityUpdate(GameManager.Entities[key].GetEntityData(), epFrom);
}
}
}
else
{
Debug.LogWarning("We sent client an confirmation but it is still sending us initial connection messages: " + epFrom.ToString());
}
}
else
{
Debug.LogWarning("Client Already connected but still sent connnection message: " + epFrom.ToString());
}
}
}
else
{
ReciveConnectionConfirmationFromHost(receivedMessage, epFrom);
}
}
public static void ProcessResponse(Message message)
{
PacketResponse response = (PacketResponse)message.Data;
mUnconfirmedPackets.Remove(response.MessageIDToConfirm);
}
public static void SendConnectionConfirmationToClient(EndPoint epFrom, PlayerData newPlayer)
{
//endpoint can be null if this is the server connecting to itself
if (epFrom != null)
{
//Tell client we have accepted its connection, and send back the player data with the updated team info
Message message = new Message(newPlayer, MessageType.Connect, GameManager.LocalPlayer.PlayerID);
//Send(message, epFrom);
byte[] data = Util.Serialize(message);
Packet newPacket = new Packet(data, message.MessageID, 1, 1, data.Length, epFrom, false);
ResponseQueue.Enqueue(newPacket);
}
}
public static void ReciveConnectionConfirmationFromHost(Message receivedMessage, EndPoint epFrom)
{
//Clear the loading win we had active
GameObject.Destroy(GameManager.LoadingWin);
PlayerData newPlayer = receivedMessage.Data as PlayerData;
GameManager.LocalPlayer.SetPlayerData(newPlayer);
IsConnected = true;
PingBack(epFrom);
}
public static void PingBack(EndPoint ep)
{
PlayerData localPlayer = GameManager.LocalPlayer.GetPlayerData();
Message message = new Message(localPlayer, MessageType.Connect, localPlayer.ID);
byte[] data = Util.Serialize(message);
Packet newPacket = new Packet(data, message.MessageID, 1, 1, data.Length, ep, false);
ResponseQueue.Enqueue(newPacket);
}
private static void SendMessageResponse(Guid packetIDToConfirm, EndPoint epFrom)
{
PacketResponse messageIDtoConfirm = new PacketResponse(packetIDToConfirm);
Message message = new Message(messageIDtoConfirm, MessageType.Response, GameManager.LocalPlayer.PlayerID);
byte[] data = Util.Serialize(message);
Packet newPacket = new Packet(data, message.MessageID, 1, 1, data.Length, epFrom, false);
ResponseQueue.Enqueue(newPacket);
}
}
[Edit]
So I continued to do research and refine my code. (code above has been updated to current version). One thing I was definitely doing wrong before was blowing up the MTU because I didn't realize the external limit was much smaller than the internal limit. Still getting the same problem though. Server can see the client message, uses the incoming endpoint to send a connection message back, and client never gets it. Currently the server will attempt to resend the connection confirmation indefinitely until the client responds back which never happens because none of the server responses ever reach the client.
My understanding of a UDP punch through is that both machines must first connect to a known external server. That external server then gives each incoming connection the others IP and port, allowing them to talk to each other directly. However, since I am manually feeding the client the IP and port of the game server, I shouldn't need an external server to do that handshake.
Client doesn't need to connect to an external server first to get the game server IP and port because I manual type in the IP and port of the game server and the game server network has the port already forwarded, so the servers address IS already known. I know for a fact that the client message is reaching the game server because server logs the incoming connection.
The game server gets the IP and port of the client from the incoming message, which is exactly what would be handed to it by the external server by the handshake. I have double checked the IP the server receives from the connection and it matches the external IP address of the client. I also verified that both machines firewalls are allowing the program through.
I would deeply appreciate some more help on figuring out why server is failing to deliver a message back to the client. Explanations or links to useful documents or research please. Telling me to do more research or look online for the answers isn't helpful. I am already doing that. The point of a questions and answers is to get help from people who already know the answers.
The problem is that there exists NAT between LAN and WLAN. You cannot just work through it so simple.
I am trying to use BluetoothConnectionService written by Mitch Tabian to connect to a Microchip BM77 dual mode module. I am trying to connect with transparent BT.
From the logcat it seems that it can not connect to the UUID and then closes socket.
Is this BM77 device not able to connect using RFcommsocket? Does it require a different UUID?
package com.example.user.bluetooth_communication;
import android.app.ProgressDialog;
import android.bluetooth.BluetoothAdapter;
import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothServerSocket;
import android.bluetooth.BluetoothSocket;
import android.content.Context;
import android.util.Log;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.UUID;
/**
* Created by User on 12/21/2016.
*/
public class BluetoothConnectionService {
private static final String TAG = "BluetoothConnectionServ";
private static final String appName = "MYAPP";
private static final UUID MY_UUID_INSECURE =
UUID.fromString("8ce255c0-200a-11e0-ac64-0800200c9a66");
private final BluetoothAdapter mBluetoothAdapter;
Context mContext;
private AcceptThread mInsecureAcceptThread;
private ConnectThread mConnectThread;
private BluetoothDevice mmDevice;
private UUID deviceUUID;
ProgressDialog mProgressDialog;
private ConnectedThread mConnectedThread;
public BluetoothConnectionService(Context context) {
mContext = context;
mBluetoothAdapter = BluetoothAdapter.getDefaultAdapter();
start();
}
/**
* This thread runs while listening for incoming connections. It behaves
* like a server-side client. It runs until a connection is accepted
* (or until cancelled).
*/
private class AcceptThread extends Thread {
// The local server socket
private final BluetoothServerSocket mmServerSocket;
public AcceptThread(){
BluetoothServerSocket tmp = null;
// Create a new listening server socket
try{
tmp = mBluetoothAdapter.listenUsingInsecureRfcommWithServiceRecord(appName, MY_UUID_INSECURE);
Log.d(TAG, "AcceptThread: Setting up Server using: " + MY_UUID_INSECURE);
}catch (IOException e){
Log.e(TAG, "AcceptThread: IOException: " + e.getMessage() );
}
mmServerSocket = tmp;
}
public void run(){
Log.d(TAG, "run: AcceptThread Running.");
BluetoothSocket socket = null;
try{
// This is a blocking call and will only return on a
// successful connection or an exception
Log.d(TAG, "run: RFCOM server socket start.....");
socket = mmServerSocket.accept();
Log.d(TAG, "run: RFCOM server socket accepted connection.");
}catch (IOException e){
Log.e(TAG, "AcceptThread: IOException: " + e.getMessage() );
}
//talk about this is in the 3rd
if(socket != null){
connected(socket,mmDevice);
}
Log.i(TAG, "END mAcceptThread ");
}
public void cancel() {
Log.d(TAG, "cancel: Canceling AcceptThread.");
try {
mmServerSocket.close();
} catch (IOException e) {
Log.e(TAG, "cancel: Close of AcceptThread ServerSocket failed. " + e.getMessage() );
}
}
}
/**
* This thread runs while attempting to make an outgoing connection
* with a device. It runs straight through; the connection either
* succeeds or fails.
*/
private class ConnectThread extends Thread {
private BluetoothSocket mmSocket;
public ConnectThread(BluetoothDevice device, UUID uuid) {
Log.d(TAG, "ConnectThread: started.");
mmDevice = device;
deviceUUID = uuid;
}
public void run(){
BluetoothSocket tmp = null;
Log.i(TAG, "RUN mConnectThread ");
// Get a BluetoothSocket for a connection with the
// given BluetoothDevice
try {
Log.d(TAG, "ConnectThread: Trying to create InsecureRfcommSocket using UUID: "
+MY_UUID_INSECURE );
tmp = mmDevice.createRfcommSocketToServiceRecord(deviceUUID);
} catch (IOException e) {
Log.e(TAG, "ConnectThread: Could not create InsecureRfcommSocket " + e.getMessage());
}
mmSocket = tmp;
// Always cancel discovery because it will slow down a connection
mBluetoothAdapter.cancelDiscovery();
// Make a connection to the BluetoothSocket
try {
// This is a blocking call and will only return on a
// successful connection or an exception
mmSocket.connect();
Log.d(TAG, "run: ConnectThread connected.");
} catch (IOException e) {
// Close the socket
try {
mmSocket.close();
Log.d(TAG, "run: Closed Socket.");
} catch (IOException e1) {
Log.e(TAG, "mConnectThread: run: Unable to close connection in socket " + e1.getMessage());
}
Log.d(TAG, "run: ConnectThread: Could not connect to UUID: " + MY_UUID_INSECURE );
}
//will talk about this in the 3rd video
connected(mmSocket,mmDevice);
}
public void cancel() {
try {
Log.d(TAG, "cancel: Closing Client Socket.");
mmSocket.close();
} catch (IOException e) {
Log.e(TAG, "cancel: close() of mmSocket in Connectthread failed. " + e.getMessage());
}
}
}
/**
* Start the chat service. Specifically start AcceptThread to begin a
* session in listening (server) mode. Called by the Activity onResume()
*/
public synchronized void start() {
Log.d(TAG, "start");
// Cancel any thread attempting to make a connection
if (mConnectThread != null) {
mConnectThread.cancel();
mConnectThread = null;
}
if (mInsecureAcceptThread == null) {
mInsecureAcceptThread = new AcceptThread();
mInsecureAcceptThread.start();
}
}
/**
AcceptThread starts and sits waiting for a connection.
Then ConnectThread starts and attempts to make a connection with the other devices AcceptThread.
**/
public void startClient(BluetoothDevice device,UUID uuid){
Log.d(TAG, "startClient: Started.");
//initprogress dialog
mProgressDialog = ProgressDialog.show(mContext,"Connecting Bluetooth"
,"Please Wait...",true);
mConnectThread = new ConnectThread(device, uuid);
mConnectThread.start();
}
/**
Finally the ConnectedThread which is responsible for maintaining the BTConnection, Sending the data, and
receiving incoming data through input/output streams respectively.
**/
private class ConnectedThread extends Thread {
private final BluetoothSocket mmSocket;
private final InputStream mmInStream;
private final OutputStream mmOutStream;
public ConnectedThread(BluetoothSocket socket) {
Log.d(TAG, "ConnectedThread: Starting.");
mmSocket = socket;
InputStream tmpIn = null;
OutputStream tmpOut = null;
//dismiss the progressdialog when connection is established
try{
mProgressDialog.dismiss();
}catch (NullPointerException e){
e.printStackTrace();
}
try {
tmpIn = mmSocket.getInputStream();
tmpOut = mmSocket.getOutputStream();
} catch (IOException e) {
e.printStackTrace();
}
mmInStream = tmpIn;
mmOutStream = tmpOut;
}
public void run(){
byte[] buffer = new byte[1024]; // buffer store for the stream
int bytes; // bytes returned from read()
// Keep listening to the InputStream until an exception occurs
while (true) {
// Read from the InputStream
try {
bytes = mmInStream.read(buffer);
String incomingMessage = new String(buffer, 0, bytes);
Log.d(TAG, "InputStream: " + incomingMessage);
} catch (IOException e) {
Log.e(TAG, "write: Error reading Input Stream. " + e.getMessage() );
break;
}
}
}
//Call this from the main activity to send data to the remote device
public void write(byte[] bytes) {
String text = new String(bytes, Charset.defaultCharset());
Log.d(TAG, "write: Writing to outputstream: " + text);
try {
mmOutStream.write(bytes);
} catch (IOException e) {
Log.e(TAG, "write: Error writing to output stream. " + e.getMessage() );
}
}
/* Call this from the main activity to shutdown the connection */
public void cancel() {
try {
mmSocket.close();
} catch (IOException e) { }
}
}
private void connected(BluetoothSocket mmSocket, BluetoothDevice mmDevice) {
Log.d(TAG, "connected: Starting.");
// Start the thread to manage the connection and perform transmissions
mConnectedThread = new ConnectedThread(mmSocket);
mConnectedThread.start();
}
/**
* Write to the ConnectedThread in an unsynchronized manner
*
* #param out The bytes to write
* #see ConnectedThread#write(byte[])
*/
public void write(byte[] out) {
// Create temporary object
ConnectedThread r;
// Synchronize a copy of the ConnectedThread
Log.d(TAG, "write: Write Called.");
//perform the write
mConnectedThread.write(out);
}
}
D/ViewRootImpl#21bc2e2[MainActivity]: ViewPostIme pointer 0
D/ViewRootImpl#21bc2e2[MainActivity]: ViewPostIme pointer 1
D/AbsListView: onTouchUp() mTouchMode : 0
D/BluetoothAdapter: cancelDiscovery
D/BluetoothAdapter: cancelDiscovery = true
D/MainActivity: onItemClick: You Clicked on a device.
D/MainActivity: onItemClick: deviceName = Dual-SPP
onItemClick: deviceAddress = 34:81:F4:40:0F:2A
Trying to pair with Dual-SPP
I/BluetoothDevice: createBond() for device 40:0 called by pid: 27565 tid: 27565
D/BluetoothConnectionServ: start
W/BluetoothAdapter: getBluetoothService() called with no BluetoothManagerCallback
D/BluetoothConnectionServ: AcceptThread: Setting up Server using: 8ce255c0-200a-11e0-ac64-0800200c9a66
D/BluetoothConnectionServ: run: AcceptThread Running.
run: RFCOM server socket start.....
D/ViewRootImpl#21bc2e2[MainActivity]: ViewPostIme pointer 0
D/ViewRootImpl#21bc2e2[MainActivity]: ViewPostIme pointer 1
D/MainActivity: startBTConnection: Initializing RFCOM Bluetooth Connection.
D/BluetoothConnectionServ: startClient: Started.
D/ScrollView: initGoToTop
D/ScrollView: initGoToTop
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: setView = DecorView#be25176[Connecting Bluetooth] TM=true MM=false
D/BluetoothConnectionServ: ConnectThread: started.
I/BluetoothConnectionServ: RUN mConnectThread
D/BluetoothConnectionServ: ConnectThread: Trying to create InsecureRfcommSocket using UUID: 8ce255c0-200a-11e0-ac64-0800200c9a66
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: dispatchAttachedToWindow
D/BluetoothAdapter: cancelDiscovery
D/BluetoothAdapter: cancelDiscovery = true
D/BluetoothUtils: isSocketAllowedBySecurityPolicy start : device null
W/BluetoothAdapter: getBluetoothService() called with no BluetoothManagerCallback
V/Surface: sf_framedrop debug : 0x4f4c, game : false, logging : 0
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: Relayout returned: old=[0,0][0,0] new=[23,790][1057,1349] result=0x7 surface={valid=true 524643803136} changed=true
D/OpenGLRenderer: eglCreateWindowSurface = 0x7a27b171e0
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: MSG_WINDOW_FOCUS_CHANGED 1
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: MSG_RESIZED_REPORT: frame=Rect(23, 790 - 1057, 1349) ci=Rect(0, 0 - 0, 0) vi=Rect(0, 0 - 0, 0) or=1
D/ViewRootImpl#21bc2e2[MainActivity]: MSG_WINDOW_FOCUS_CHANGED 0
D/BluetoothSocket: close() this: android.bluetooth.BluetoothSocket#3daba6f, channel: -1, mSocketIS: android.net.LocalSocketImpl$SocketInputStream#9c9637c, mSocketOS: android.net.LocalSocketImpl$SocketOutputStream#4f75405mSocket: android.net.LocalSocket#9c67c5a impl:android.net.LocalSocketImpl#b028e8b fd:java.io.FileDescriptor#2073b68, mSocketState: INIT
D/BluetoothConnectionServ: run: Closed Socket.
D/BluetoothConnectionServ: run: ConnectThread: Could not connect to UUID: 8ce255c0-200a-11e0-ac64-0800200c9a66
connected: Starting.
ConnectedThread: Starting.
D/BluetoothUtils: isSocketAllowedBySecurityPolicy start : device null
D/BluetoothUtils: isSocketAllowedBySecurityPolicy start : device null
D/OpenGLRenderer: eglDestroySurface = 0x7a27b171e0
D/ViewRootImpl#fcf1c11[Connecting Bluetooth]: dispatchDetachedFromWindow
D/InputEventReceiver: channel '48033aa Connecting Bluetooth (client)' ~ Disposing input event receiver.
D/InputEventReceiver: channel '48033aa Connecting Bluetooth (client)' ~NativeInputEventReceiver.
D/ViewRootImpl#21bc2e2[MainActivity]: MSG_WINDOW_FOCUS_CHANGED 1
E/ViewRootImpl: sendUserActionEvent() returned.
E/BluetoothConnectionServ: write: Error reading Input Stream. socket closed
I am trying to connect raspberry pi 2 windows IoT using usb to serial ttl module on headless mode. I couldn' t find any working example. I just want o send "hello world" to raspberry pi. Microsoft example is too complicated and also you have to click connect button to send and receive data. I tried to change code but I failed to run example. https://ms-iot.github.io/content/en-US/win10/samples/SerialSample.htm
I made a class for send and receive from serial port
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Windows.Storage.Streams;
using Windows.Devices.SerialCommunication;
using System.Threading;
using Windows.Devices.Enumeration;
namespace OMEGA.Raspberry
{
public delegate void DataRecievedDlg(byte[] data);
class USART
{
#region Private Vars
DataReader dataReaderObject = null;
private CancellationTokenSource ReadCancellationTokenSource;
private uint _baudrate;
private SerialParity _parity;
private SerialStopBitCount _stopBits;
private ushort _dataBits;
#endregion
#region Public Vars
public SerialDevice serialPort { get; private set; }
public event DataRecievedDlg DataRecieved;
#endregion
#region Private Functions
public USART()
{
serialPort = null;
}
private async void Listen()
{
try
{
if (serialPort != null)
{
dataReaderObject = new DataReader(serialPort.InputStream);
// keep reading the serial input
while (true)
{
byte[] x = await ReadAsync(ReadCancellationTokenSource.Token);
if (x.Length > 0)
DataRecieved(x);
}
}
}
catch (Exception ex)
{
if (ex.GetType().Name != "TaskCanceledException")
//{
// CloseDevice();
//}
//else
{
throw new Exception(ex.Message);
}
}
finally
{
// Cleanup once complete
if (dataReaderObject != null)
{
dataReaderObject.DetachStream();
dataReaderObject = null;
}
}
}
private async Task<byte[]> ReadAsync(CancellationToken cancellationToken)
{
Task<UInt32> loadAsyncTask;
uint ReadBufferLength = 1024;
// If task cancellation was requested, comply
cancellationToken.ThrowIfCancellationRequested();
// Set InputStreamOptions to complete the asynchronous read operation when one or more bytes is available
dataReaderObject.InputStreamOptions = InputStreamOptions.Partial;
// Create a task object to wait for data on the serialPort.InputStream
loadAsyncTask = dataReaderObject.LoadAsync(ReadBufferLength).AsTask(cancellationToken);
// Launch the task and wait
UInt32 bytesRead = await loadAsyncTask;
byte[] bytes = new byte[bytesRead];
if (bytesRead > 0)
dataReaderObject.ReadBytes(bytes);
return bytes;
}
#endregion
#region Public Functions
public void CloseDevice()
{
if (serialPort != null)
{
serialPort.Dispose();
}
serialPort = null;
}
public void StopListening()
{
if (ReadCancellationTokenSource != null)
{
if (!ReadCancellationTokenSource.IsCancellationRequested)
{
ReadCancellationTokenSource.Cancel();
}
}
}
public async Task Initialize(uint baudrate, SerialParity parity, SerialStopBitCount stopBits, ushort dataBits)
{
try
{
string aqs = SerialDevice.GetDeviceSelector();
var dis = await DeviceInformation.FindAllAsync(aqs);
serialPort = await SerialDevice.FromIdAsync(dis[0].Id);
serialPort.WriteTimeout = TimeSpan.FromMilliseconds(1000);
serialPort.ReadTimeout = TimeSpan.FromMilliseconds(1000);
serialPort.BaudRate = baudrate;
serialPort.Parity = parity;
serialPort.StopBits = stopBits;
serialPort.DataBits = dataBits;
serialPort.Handshake = SerialHandshake.None;
_baudrate = baudrate;
_parity = parity;
_stopBits = stopBits;
_dataBits = dataBits;
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
public void StartListening()
{
ReadCancellationTokenSource = new CancellationTokenSource();
Listen();
}
public async Task<uint> WriteAsync(byte[] data)
{
DataWriter dataWriter = new DataWriter();
dataWriter.WriteBytes(data);
uint bytesWritten = await serialPort.OutputStream.WriteAsync(dataWriter.DetachBuffer());
return bytesWritten;
}
#endregion
}
}
I am using netty 3.5.11 with Jdk 1.7 on Ubuntu to receive a large amount of updates of stocks rates at a very high frequency. The message format being sent is JSON. The data is subscribed from topic on a redis server. There is a Subscriber for each symbol. The channel object is passed to multiple Subscribers and on receiving the data it is written to the client.
Now the amount of data received is around 25,000 records in 2 minutes. Each record size is on an average around 500 bytes long.
During test runs around 7500/8000 records were dropped because the channel was not writable.
How do i avoid this. ?
I also noticed that the latency increases systematically leading to updates being received after a long period. This happened when i I used Bufferedwritehandler to avoid packet drops.
Here are the options that i set on bootstrap.
executionHandler = new ExecutionHandler(
new OrderedMemoryAwareThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, 1000000, 10000000, 100,
TimeUnit.MILLISECONDS));
serverBootStrap.setPipelineFactory(new ChannelPipelineFactory()
{
#Override
public ChannelPipeline getPipeline() throws Exception
{
return Channels.pipeline(new PortUnificationServerHandler(getConfiguration(), executionHandler));
}
});
serverBootStrap.setOption("child.tcpNoDelay", true);
serverBootStrap.setOption("tcpNoDelay", true);
serverBootStrap.setOption("child.keepAlive", true);
serverBootStrap.setOption("child.reuseAddress", true);
//setting buffer size can improve I/O
serverBootStrap.setOption("child.sendBufferSize", 16777216);
serverBootStrap.setOption("receiveBufferSize", 16777216);//1048576);
// better to have an receive buffer predictor
serverBootStrap.setOption("receiveBufferSizePredictorFactory", new AdaptiveReceiveBufferSizePredictorFactory(1024, 1024 * 16, 16777216));//1048576));
//if the server is sending 1000 messages per sec, optimum write buffer water marks will
//prevent unnecessary throttling, Check NioSocketChannelConfig doc
serverBootStrap.setOption("backlog", 1000);
serverBootStrap.setOption("sendBufferSize", 16777216);//1048576);
serverBootStrap.setOption("writeBufferLowWaterMark", 1024 * 1024 * 25);
serverBootStrap.setOption("writeBufferHighWaterMark", 1024 * 1024 * 50);
The pipeline and handlers class
public class PortUnificationServerHandler extends FrameDecoder
{
private AppConfiguration appConfiguration;
private final ExecutionHandler executionHandler;
public PortUnificationServerHandler(AppConfiguration pAppConfiguration, ExecutionHandler pExecutionHandler)
{
appConfiguration = pAppConfiguration;
this.executionHandler = pExecutionHandler;
}
#Override
protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception
{
String lRequest = buffer.toString(CharsetUtil.UTF_8);
if (ConnectionServiceHelper.isValidJSON(lRequest))
{
ObjectMapper lObjectMapper = new ObjectMapper();
StringReader lStringReader = new StringReader(lRequest);
JsonNode lNode = lObjectMapper.readTree(lStringReader);
if (lNode.get(Constants.REQUEST_TYPE).asText().trim().equalsIgnoreCase(Constants.LOGIN_REQUEST))
{
JsonNode lDataNode1 = lNode.get(Constants.REQUEST_DATA);
LoginRequest lLogin = lObjectMapper.treeToValue(lDataNode1, LoginRequest.class);
if (lLogin.getCompress() != null)
{
if (lLogin.getCompress().trim().equalsIgnoreCase(Constants.COMPRESS_FLAG_TRUE))
{
enableJSON(ctx);
enableGzip(ctx);
ctx.getPipeline().remove(this);
}
else
{
enableJSON(ctx);
ctx.getPipeline().remove(this);
}
}
else
{
enableJSON(ctx);
ctx.getPipeline().remove(this);
}
}
}
// Forward the current read buffer as is to the new handlers.
return buffer.readBytes(buffer.readableBytes());
}
private void enableJSON(ChannelHandlerContext ctx)
{
ChannelPipeline pipeline = ctx.getPipeline();
boolean lHandlerExists = pipeline.getContext("bufferedwriter") != null;
if (!lHandlerExists)
{
pipeline.addFirst("bufferedwriter", new MyBufferedWriteHandler()); // 80960
}
boolean lHandlerExists = pipeline.getContext("framer") != null;
if (!lHandlerExists)
{
pipeline.addLast("framer", new DelimiterBasedFrameDecoder(65535,
new ChannelBuffer[]
{
ChannelBuffers.wrappedBuffer(
new byte[]
{
'\n'
})
}));
}
lHandlerExists = pipeline.getContext("decoder") != null;
if (!lHandlerExists)
{
pipeline.addLast("decoder", new StringDecoder(CharsetUtil.UTF_8));
}
lHandlerExists = pipeline.getContext("encoder") != null;
if (!lHandlerExists)
{
pipeline.addLast("encoder", new StringEncoder(CharsetUtil.UTF_8));
}
lHandlerExists = pipeline.getContext("executor") != null;
if (!lHandlerExists)
{
pipeline.addLast("executor", executionHandler);
}
lHandlerExists = pipeline.getContext("handler") != null;
if (!lHandlerExists)
{
pipeline.addLast("handler", new ConnectionServiceUpStreamHandler(appConfiguration));
}
lHandlerExists = pipeline.getContext("unite") != null;
if (!lHandlerExists)
{
pipeline.addLast("unite", new PortUnificationServerHandler(appConfiguration, executionHandler));
}
}
private void enableGzip(ChannelHandlerContext ctx)
{
ChannelPipeline pipeline = ctx.getPipeline();
//pipeline.remove("decoder");
//pipeline.addLast("decoder", new MyStringDecoder(CharsetUtil.UTF_8, true));
//pipeline.addLast("compress", new CompressionHandler(80, "gzipdeflater"));
boolean lHandlerExists = pipeline.getContext("encoder") != null;
if (lHandlerExists)
{
pipeline.remove("encoder");
}
lHandlerExists = pipeline.getContext("gzipdeflater") != null;
if (!lHandlerExists)
{
pipeline.addBefore("executor", "gzipdeflater", new ZlibEncoder(ZlibWrapper.GZIP));
}
lHandlerExists = pipeline.getContext("lengthprepender") != null;
if (!lHandlerExists)
{
pipeline.addAfter("gzipdeflater", "lengthprepender", new LengthFieldPrepender(4));
}
}
}
The BufferedWriterHandler
public class MyBufferedWriteHandler extends BufferedWriteHandler
{
private final AtomicLong bufferSize = new AtomicLong();
final Logger logger = LoggerFactory.getLogger(getClass());
public MyBufferedWriteHandler() {
// Enable consolidation by default.
super(true);
}
#Override
public void writeRequested(ChannelHandlerContext ctx, MessageEvent e) throws Exception
{
ChannelBuffer data = (ChannelBuffer) e.getMessage();
if (e.getChannel().isWritable())
{
long newBufferSize = bufferSize.get();
// Flush the queue if it gets larger than 8KiB.
if (newBufferSize > 0)
{
flush();
bufferSize.set(0);
}
ctx.sendDownstream(e);
}
else
{
logger.warn( "Buffering data for : " + e.getChannel().getRemoteAddress() );
super.writeRequested(ctx, e);
bufferSize.addAndGet(data.readableBytes());
}
}
#Override
public void channelInterestChanged(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception
{
if (e.getChannel().isWritable())
{
flush();
}
}
The function used in the Subscriber class to write data
public void writeToClient(Channel pClientChannel, String pMessage) throws IOException
{
String lMessage = pMessage;
if (pClientChannel.isWritable())
{
lMessage += Constants.RESPONSE_DELIMITER;
pClientChannel.write(lMessage);
}
else
{
logger.warn(DroppedCounter++ + " droppped : " + pMessage);
}
}
I have implemented some of the suggestions that i read on stackoverflow and other sites. But i have not been successfull in resolving this issue.
Kindly suggest or advice as to what am i missing ?
Thanks
Under here you find a simple server/client app that sends the integer 5 from the client to the server, which reads it and sends it back to the client. On the server I have placed a latency meter around the DataInputStream.readInt() method, which reads that this method is causing a 400 ms latency.
Server code:
import java.io.*;
import java.net.*;
public class Server {
public static void main(String args[]) {
// declaration section:
ServerSocket echoServer = null;
Socket clientSocket = null;
int x;
// Try to open a server socket on port 4444
try {
echoServer = new ServerSocket(4444);
}
catch (IOException e) {
System.out.println(e);
}
//accept connection
try {
clientSocket = echoServer.accept();
//input
DataInputStream input = new DataInputStream(clientSocket.getInputStream());
//output
DataOutputStream output = new DataOutputStream(clientSocket.getOutputStream());
//loop
while(true){
long time = System.currentTimeMillis();
output.writeInt(input.readInt());
long dtime = System.currentTimeMillis();
System.out.println(dtime-time);
}
//close
// output.close();
// input.close();
// clientSocket.close();
// echoServer.close();
}catch (IOException e) {
System.out.println(e);
}
}
}
Client code:
import java.io.*;
import java.net.*;
public class Client {
public static void main(String[] args) {
// declaration section:
// clientClient: our client socket
// out: output stream
// in: input stream
Socket clientSocket = null;
// Initialization section:
// Try to open a socket on port 4444
// Try to open input and output streams
try{
clientSocket = new Socket("YOUR-IP-HERE", 4444);
//output
DataOutputStream output = new DataOutputStream(clientSocket.getOutputStream());
//input
DataInputStream input = new DataInputStream(clientSocket.getInputStream());
//loop
while(true) {
output.writeInt(5);
//System.out.println(input.readInt());
}
//close
// output.close();
// input.close();
// clientSocket.close();
}catch(Exception e) {
System.out.println("error");
}
}
}
Problem area:
//latencymeter
long time = System.currentTimeMillis();
//problem code
output.writeInt(input.readInt());
//latencymeter
long dtime = System.currentTimeMillis();
System.out.println(dtime-time);
Am I making a mistake in my code or am I not coding efficiently, please let me know.
Thanks
Solved: Adding clientSocket.setTcpNoDelay(true); on both server and client has brought the latency down to a normal 10-20 ms.