CSV File Not Being Read from Servlet - servlets

As the title says, I have a CSV file that will not load from within my web application. I am using Netbeans to build the project.
Whenever I launch the project from Netbeans, it works like it should however when I take the war file and try to deploy it from within the Glassfish interface it shows the variables as undefined which tells me that it is not reading the file. Screenshots below show what is happening and my folder structure.
I have read many posts here and #BalusC has some great information here, but its not working for me and I believe this is somehow my fault, but I need a bit more specific help here than just reading another post.
I have put the CSV file that I am intending to load into the /src/main/resources folder as noted here by BalusC. The code I am using to load the file is as follows.
As a side note, I have a JSP that I am using to check the location and access to the file. The JSP can access and display the file without any problems when the application is deployed manually.
Edit: I ran a debug and could not find anything wrong, so I ran glassfish in verbose mode and loaded the page, once the page was up, it started reading from the file and sending the data but still shows "undefined" in all fields.
Here is the output data from running glassfish in verbose mode.
[#|2017-05-05T16:34:37.609+0900|INFO|glassfish 4.1|DukeETFServlet|_ThreadID=33;_ThreadName=http-listener-1(3);_TimeMillis=1493969677609;_LevelValue=800;|
Connection open.|#]
[#|2017-05-05T16:34:38.014+0900|INFO|glassfish 4.1|DukeETFServlet|_ThreadID=109;_ThreadName=__ejb-thread-pool3;_TimeMillis=1493969678014;_LevelValue=800;|
Sent: ABRN / Arbor Realty Trust 7.375% Senior / 25.32 / 25.11 / 25.24 / 12000 / 24.27 / 26.15 / Fri May 05 16:34:38 JST 2017|#]
[#|2017-05-05T16:34:38.016+0900|INFO|glassfish 4.1|DukeETFServlet|_ThreadID=109;_ThreadName=__ejb-thread-pool3;_TimeMillis=1493969678016;_LevelValue=800;|
Connection closed.|#]
[#|2017-05-05T16:34:38.024+0900|INFO|glassfish 4.1|DukeETFServlet|_ThreadID=34;_ThreadName=http-listener-1(4);_TimeMillis=1493969678024;_LevelValue=800;|
Connection open.|#]
[#|2017-05-05T16:34:38.029+0900|INFO|glassfish 4.1|DukeETFServlet|_ThreadID=119;_ThreadName=__ejb-thread-pool4;_TimeMillis=1493969678029;_LevelValue=800;|
Sent: ABT / Abbott Laboratories / 44.01 / 43.60 / 43.65 / 7487400 / 36.76 / 45.84 / Fri May 05 16:34:38 JST 2017|#]
Here is the data for loading the file.
Servlet
#WebServlet(urlPatterns={"/dukeetf"}, asyncSupported=true)
public class DukeETFServlet extends HttpServlet {
private static final Logger logger = Logger.getLogger("DukeETFServlet");
private static final long serialVersionUID = 2114153638027156979L;
private Queue<AsyncContext> requestQueue;
#EJB private PriceVolumeBean pvbean;
#Override
public void init(ServletConfig config) {
/* Queue for requests */
requestQueue = new ConcurrentLinkedQueue<>();
/* Register with the bean that provides price/volume updates */
pvbean.registerServlet(this);
}
/* PriceVolumeBean calls this method every second to send updates */
public void send(String ticker, String name, float highPrice, float lowPrice,
float closingPrice, int volume, float fiftyTwoWeekHigh, float fiftyTwoWeekLow,
String currentTime) {
/* Send update to all connected clients */
for (AsyncContext acontext : requestQueue) {
try {
String msg = String.format("%s / %s / %.2f / %.2f / %.2f / %d /"
+ " %.2f / %.2f / %s",
ticker, name, highPrice, lowPrice, closingPrice, volume,
fiftyTwoWeekHigh, fiftyTwoWeekLow, currentTime);
PrintWriter writer = acontext.getResponse().getWriter();
writer.write(msg);
logger.log(Level.INFO, "Sent: {0}", msg);
/* Close the connection
* The client (JavaScript) makes a new one instantly */
acontext.complete();
} catch (IOException ex) {
logger.log(Level.INFO, ex.toString());
}
}
}
/* Service method */
#Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html");
/* Put request in async mode. */
final AsyncContext acontext = request.startAsync();
/* Remove from the queue when done */
acontext.addListener(new AsyncListener() {
#Override
public void onComplete(AsyncEvent ae) throws IOException {
requestQueue.remove(acontext);
logger.log(Level.INFO, "Connection Being Closed.");
}
#Override
public void onTimeout(AsyncEvent ae) throws IOException {
requestQueue.remove(acontext);
logger.log(Level.INFO, "Connection Has Timed Out.");
}
#Override
public void onError(AsyncEvent ae) throws IOException {
requestQueue.remove(acontext);
logger.log(Level.INFO, "Connection error.");
}
#Override
public void onStartAsync(AsyncEvent ae) throws IOException { }
});
/* Add to the queue */
requestQueue.add(acontext);
logger.log(Level.INFO, "Connection Being Opened.");
}
}
Class to get information from CSV
//Get Stock Data From CSV File
public static ArrayList<Stock> getListOfStocks() throws IOException {
ArrayList<Stock> stocks = new ArrayList();
ClassLoader classLoader =
Thread.currentThread().getContextClassLoader();
InputStream is =
StockService.class.getResourceAsStream("/stockdata.csv");
// create an instance of BufferedReader
// using try with resource, Java 7 feature to close resources
try (CSVReader reader = new CSVReader(new InputStreamReader(is))) {
// read the first line from the text file
String[] nextLine;
reader.readNext();
// loop until all lines are read
while ((nextLine = reader.readNext()) != null) {
Stock newStock = new Stock(nextLine[0], nextLine[1],
Float.valueOf(nextLine[2]), Float.valueOf(nextLine[3]),
Float.valueOf(nextLine[4]), Integer.valueOf(nextLine[5]),
Float.valueOf(nextLine[6]), Float.valueOf(nextLine[7]));
stocks.add(newStock);
}
}
return stocks;
}
Bean that retrieves and sends information
/* Updates price and volume information every second */
#Startup
#Singleton
public class PriceVolumeBean {
/* Use the container's timer service */
#Resource TimerService tservice;
private DukeETFServlet servlet;
//Set Variable for Counter
private int i = 0;
//Set date time variable
String currentTime;
//Set Variables for Stock Data
private String ticker;
private String name;
private float highPrice;
private float lowPrice;
private float closingPrice;
private int volume;
private float fiftyTwoWeekHigh;
private float fiftyTwoWeekLow;
private static final Logger logger = Logger.getLogger("PriceVolumeBean");
#PostConstruct
public void init() {
/* Intialize the EJB and create a timer */
logger.log(Level.INFO, "Initializing EJB.");
servlet = null;
tservice.createIntervalTimer(2000, 2000, new TimerConfig());
}
public void registerServlet(DukeETFServlet servlet) {
/* Associate a servlet to send updates to */
this.servlet = servlet;
}
#Timeout
public void timeout() throws IOException {
// Update Date
Date date = new Date();
// Set stock variables //
ticker = StockService.getListOfStocks().get(i).getTicker();
name = StockService.getListOfStocks().get(i).getName();
highPrice = StockService.getListOfStocks().get(i).getHighPrice();
lowPrice = StockService.getListOfStocks().get(i).getLowPrice();
closingPrice = StockService.getListOfStocks().get(i).getClosingPrice();
volume = StockService.getListOfStocks().get(i).getVolume();
fiftyTwoWeekHigh = StockService.getListOfStocks().get(i).getFiftyTwoWeekHigh();
fiftyTwoWeekLow = StockService.getListOfStocks().get(i).getFiftyTwoWeekLow();
currentTime = date.toString();
// Send updated information
if (servlet != null)
servlet.send(ticker, name, highPrice, lowPrice, closingPrice,
volume, fiftyTwoWeekHigh, fiftyTwoWeekLow, currentTime);
// Counter that keeps from going beyond size of arraylist
i++;
if (i == 100) {
i = 0;
}
}
}

Related

Unit test for post sling servlet aem 6.5

I have the following POST servlet that adds new node under certain resource with parameters(name and last nam) from the request:
#Component(
service = Servlet.class,
property = {
"sling.servlet.paths=/bin/createuser",
"sling.servlet.methods=" + HttpConstants.METHOD_POST
})
public class CreateNodeServlet extends SlingAllMethodsServlet {
/**
* Logger
*/
private static final Logger log = LoggerFactory.getLogger(CreateNodeServlet.class);
#Override
protected void doPost(final SlingHttpServletRequest req, final SlingHttpServletResponse resp) throws IOException {
log.info("Inside CreateNodeServlet");
ResourceResolver resourceResolver = req.getResourceResolver();
final Resource resource = resourceResolver.getResource("/content/test/us/en");
String name = req.getParameter("name");
String lastname = req.getParameter("lastname");
log.info("name :{}",name);
log.info("lastname :{}",lastname);
Node node = resource.adaptTo(Node.class);
try {
log.info("Node {}", node.getName() );
Node newNode = node.addNode(name+lastname, "nt:unstructured");
newNode.setProperty("name", name);
newNode.setProperty("lastname", lastname);
resourceResolver.commit();
} catch (RepositoryException e) {
e.printStackTrace();
} catch (PersistenceException e) {
e.printStackTrace();
}
resp.setStatus(200);
resp.getWriter().write("Simple Post Test");
}
}
I tried creating unit test for this I got this so far:
#ExtendWith(AemContextExtension.class)
class CreateNodeServletTest {
private final AemContext context = new AemContext();
private CreateNodeServlet createNodeServlet = new CreateNodeServlet();
#Test
void doPost() throws IOException, JSONException {
context.currentPage(context.pageManager().getPage("/bin/createuser"));
context.currentResource(context.resourceResolver().getResource("/bin/createuser"));
context.requestPathInfo().setResourcePath("/bin/createuser");
MockSlingHttpServletRequest request = context.request();
MockSlingHttpServletResponse response = context.response();
createNodeServlet.doPost(request, response);
JSONArray output = new JSONArray(context.response().getOutputAsString());
assertEquals("Simple Post Test", output);
}
}
however this is not working I am getting null pointer on this line
Node node = resource.adaptTo(Node.class);
can some one help what I am missing and some tips will be of great help as I am new to AEM, and there is not much resources about unit testing sling servlets ?
I think you need to register JCR_MOCK as resource resolver type
new AemContext(ResourceResolverType.JCR_MOCK);

Netty: TCP Client Server File transfer: Exception TooLongFrameException:

I am new to netty and I am trying to design a solution as below for transfer of file from Server to Client over TCP:
1. Zero copy based file transfer in case of non-ssl based transfer (Using default region of the file)
2. ChunkedFile transfer in case of SSL based transfer.
The Client - Server file transfer works in this way:
1. The client sends the location of the file to be transfered
2. Based on the location (sent by the client) the server transfers the file to the client
The file content could be anything (String /image /pdf etc) and any size.
Now, I get this TooLongFrameException: at the Server side, though the server is just decoding the path received from the client, for running the code mentioned below (Server/Client).
io.netty.handler.codec.TooLongFrameException: Adjusted frame length exceeds 65536: 215542494061 - discarded
at io.netty.handler.codec.LengthFieldBasedFrameDecoder.fail(LengthFieldBasedFrameDecoder.java:522)
at io.netty.handler.codec.LengthFieldBasedFrameDecoder.failIfNecessary(LengthFieldBasedFrameDecoder.java:500)
Now, My question is:
Am I wrong with the order of Encoders and Decoders and its configuration? If so, what is the correct way to configure it to receive a file from the server?
I went through few related StackOverflow posts SO Q1,SO Q2 , SO Q3, SO Q4. I got to know about the LengthFieldBasedDecoder, but I didn't get to know how to configure its corresponding LengthFieldPrepender at the Server (Encoding side). Is it even required at all?
Please point me into the right direction.
FileClient:
public final class FileClient {
static final boolean SSL = System.getProperty("ssl") != null;
static final int PORT = Integer.parseInt(System.getProperty("port", SSL ? "8992" : "8023"));
static final String HOST = System.getProperty("host", "127.0.0.1");
public static void main(String[] args) throws Exception {
// Configure SSL.
final SslContext sslCtx;
if (SSL) {
SelfSignedCertificate ssc = new SelfSignedCertificate();
sslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build();
} else {
sslCtx = null;
}
// Configure the client
EventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioSocketChannel.class)
.option(ChannelOption.SO_KEEPALIVE, true)
.handler(new ChannelInitializer<SocketChannel>() {
#Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
if (sslCtx != null) {
pipeline.addLast(sslCtx.newHandler(ch.alloc(), HOST, PORT));
}
pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(64*1024, 0, 8));
pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
pipeline.addLast(new ObjectDecoder(ClassResolvers.cacheDisabled(null)));
pipeline.addLast(new ObjectEncoder());
pipeline.addLast( new FileClientHandler()); }
});
// Start the server.
ChannelFuture f = b.connect(HOST,PORT).sync();
// Wait until the server socket is closed.
f.channel().closeFuture().sync();
} finally {
// Shut down all event loops to terminate all threads.
group.shutdownGracefully();
}
}
}
FileClientHandler:
public class FileClientHandler extends ChannelInboundHandlerAdapter{
#Override
public void channelActive(ChannelHandlerContext ctx) {
String filePath = "/Users/Home/Documents/Data.pdf";
ctx.writeAndFlush(Unpooled.wrappedBuffer(filePath.getBytes()));
}
#Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
System.out.println("File Client Handler Read method...");
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
cause.printStackTrace();
}
}
FileServer:
/**
* Server that accept the path of a file and echo back its content.
*/
public final class FileServer {
static final boolean SSL = System.getProperty("ssl") != null;
static final int PORT = Integer.parseInt(System.getProperty("port", SSL ? "8992" : "8023"));
public static void main(String[] args) throws Exception {
// Configure SSL.
final SslContext sslCtx;
if (SSL) {
SelfSignedCertificate ssc = new SelfSignedCertificate();
sslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build();
} else {
sslCtx = null;
}
// Configure the server.
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_KEEPALIVE, true).handler(new LoggingHandler(LogLevel.INFO))
.childHandler(new ChannelInitializer<SocketChannel>() {
#Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
if (sslCtx != null) {
pipeline.addLast(sslCtx.newHandler(ch.alloc()));
}
pipeline.addLast("frameDecoder",new LengthFieldBasedFrameDecoder(64*1024, 0, 8));
pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
pipeline.addLast(new ObjectDecoder(ClassResolvers.cacheDisabled(null)));
pipeline.addLast(new ObjectEncoder());
pipeline.addLast(new ChunkedWriteHandler());
pipeline.addLast(new FileServerHandler());
}
});
// Start the server.
ChannelFuture f = b.bind(PORT).sync();
// Wait until the server socket is closed.
f.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
}
}
FileServerHandler:
public class FileServerHandler extends ChannelInboundHandlerAdapter {
#Override
public void channelRead(ChannelHandlerContext ctx, Object obj) throws Exception {
RandomAccessFile raf = null;
long length = -1;
try {
ByteBuf buff = (ByteBuf)obj;
byte[] bytes = new byte[buff.readableBytes()];
buff.readBytes(bytes);
String msg = new String(bytes);
raf = new RandomAccessFile(msg, "r");
length = raf.length();
} catch (Exception e) {
ctx.writeAndFlush("ERR: " + e.getClass().getSimpleName() + ": " + e.getMessage() + '\n');
return;
} finally {
if (length < 0 && raf != null) {
raf.close();
}
}
if (ctx.pipeline().get(SslHandler.class) == null) {
// SSL not enabled - can use zero-copy file transfer.
ctx.writeAndFlush(new DefaultFileRegion(raf.getChannel(), 0, length));
} else {
// SSL enabled - cannot use zero-copy file transfer.
ctx.writeAndFlush(new ChunkedFile(raf));
}
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
cause.printStackTrace();
System.out.println("Exception server.....");
}
}
I referred Netty In Action and code samples from here
There are multiple things wrong with your server/client. First thing the SSL, for the client you don't need to initialize a SslContext for a server instead you would do something like this:
sslCtx = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();
On the server side of things you use a SelfSignedCertificate which in itself isn't wrong but would like to remind you that it should only be used for debugging purposes and not in production. In addition you use the ChannelOption.SO_KEEPALIVE which isn't recommended since the keepalive interval is OS-dependent. Furthermore you added Object En-/Decoder to your pipeline which in your case don't do anything useful so you can remove them.
Also you configured your LengthFieldBasedFrameDecoder wrong due to an incomplete and wrong parameter list. In the netty docs you need the version of the constructor which defines the lengthFieldLength and initialBytesToStrip. Besides the not stripping the length field you also defined the wrong lengthFieldLength which should be the same as your LengthFieldPrepender's lengthFieldLength which is 4 bytes. In conlusion you could use the constructor like this:
new LengthFieldBasedFrameDecoder(64 * 1024, 0, 4, 0, 4)
In both your handler you don't specify a Charset when en-/decoding your String which could lead to problems because if no ´Charset´ is defined the systems default will be used which could vary. You could do something like this:
//to encode the String
string.getBytes(StandardCharsets.UTF_8);
//to decode the String
new String(bytes, StandardCharsets.UTF_8);
Additionally you tried to use the DefaultFileRegion if no SslHandler was added to the pipeline which would have been fine if you didn't added the LengthFieldHandler since they would need a memory copy of the byte[] to send to added the length field. Moreover I would recommend using the ChunkedNioFile instead of the ChunkedFile because it's nonblocking which is always a good thing. You would do this like that:
new ChunkedNioFile(randomAccessFile.getChannel())
One final thing on how to decode a ChunkedFile as it's split in chunks you can simply assamble them tougether with a simple OutputStream. Here's an old file handler of mine:
public class FileTransferHandler extends SimpleChannelInboundHandler<ByteBuf> {
private final Path path;
private final int size;
private final int hash;
private OutputStream outputStream;
private int writtenBytes = 0;
private byte[] buffer = new byte[0];
protected FileTransferHandler(Path path, int size, int hash) {
this.path = path;
this.size = size;
this.hash = hash;
}
#Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception {
if(this.outputStream == null) {
Files.createDirectories(this.path.getParent());
if(Files.exists(this.path))
Files.delete(this.path);
this.outputStream = Files.newOutputStream(this.path, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
int size = byteBuf.readableBytes();
if(size > this.buffer.length)
this.buffer = new byte[size];
byteBuf.readBytes(this.buffer, 0, size);
this.outputStream.write(this.buffer, 0, size);
this.writtenBytes += size;
if(this.writtenBytes == this.size && MurMur3.hash(this.path) != this.hash) {
System.err.println("Received file has wrong hash");
return;
}
}
#Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
if(this.outputStream != null)
this.outputStream.close();
}
}

Can Storm's HdfsBolt flush data after a timeout as well?

We are using Storm to process streaming data and store into HDFS. We have got everything to work but have one issue. I understand that we can specify the number of tuples after which the data gets flushed to HDFS using SyncPolicy, something like this below:
SyncPolicy syncPolicy = new CountSyncPolicy(Integer.parseInt(args[3]));
The question I have is can the data also be flushed after a timeout? For e.g. we have set the SyncPolicy above to 1000 tuples. If for whatever reason we get 995 tuples and then the data stops coming in for a while is there any way that storm can flush the 995 records to HDFS after a specified timeout (5 seconds)?
Thanks in advance for any help on this!
Shay
Yes, if you send a tick tuple to the HDFS bolt, it will cause the bolt to try to sync to the HDFS file system. All this happens in the HDFS bolt's execute function.
To configure tick tuples for your topology, in your topology config. In Java, to set that to every 300 seconds the code would look like:
Config topologyConfig = new Config();
topologyConfig.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 300);
StormSubmitter.submitTopology("mytopology", topologyConfig, builder.createTopology());
You'll have to adjust that last line depending on your circumstances.
There is an alternative solution for this problem,
First, lets clarify about sync policy, If your sync policy is 1000 ,then HdfsBolt only sync the data from 1000 tuple by calling hsync() method in execute() means it only clears the buffer by pushing data to disk, but for faster write disk may uses its cache and not writing to file directly.
The data is written to the file only when the size of data matches your rotation policy that need to specify at the time of bolt creation.
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(100.0f, Units.KB);
So for flushing the record the to file after timeout, Seperate your tick tuple from normal tuples in excecute method and calculate the time difference of both tuple, If the diff is greater than timeout period then write the data to file.
By handling tick tuple differently you can also avoid the tick tuple frequency written to your file.
See the below code for better understanding:
public class CustomHdfsBolt1 extends AbstractHdfsBolt {
private static final Logger LOG = LoggerFactory.getLogger(CustomHdfsBolt1.class);
private transient FSDataOutputStream out;
private RecordFormat format;
private long offset = 0L;
private int tickTupleCount = 0;
private String type;
private long normalTupleTime;
private long tickTupleTime;
public CustomHdfsBolt1() {
}
public CustomHdfsBolt1(String type) {
this.type = type;
}
public CustomHdfsBolt1 withFsUrl(String fsUrl) {
this.fsUrl = fsUrl;
return this;
}
public CustomHdfsBolt1 withConfigKey(String configKey) {
this.configKey = configKey;
return this;
}
public CustomHdfsBolt1 withFileNameFormat(FileNameFormat fileNameFormat) {
this.fileNameFormat = fileNameFormat;
return this;
}
public CustomHdfsBolt1 withRecordFormat(RecordFormat format) {
this.format = format;
return this;
}
public CustomHdfsBolt1 withSyncPolicy(SyncPolicy syncPolicy) {
this.syncPolicy = syncPolicy;
return this;
}
public CustomHdfsBolt1 withRotationPolicy(FileRotationPolicy rotationPolicy) {
this.rotationPolicy = rotationPolicy;
return this;
}
public CustomHdfsBolt1 addRotationAction(RotationAction action) {
this.rotationActions.add(action);
return this;
}
protected static boolean isTickTuple(Tuple tuple) {
return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)
&& tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}
public void execute(Tuple tuple) {
try {
if (isTickTuple(tuple)) {
tickTupleTime = Calendar.getInstance().getTimeInMillis();
long timeDiff = normalTupleTime - tickTupleTime;
long diffInSeconds = TimeUnit.MILLISECONDS.toSeconds(timeDiff);
if (diffInSeconds > 5) { // specify the value you want.
this.rotateWithOutFileSize(tuple);
}
} else {
normalTupleTime = Calendar.getInstance().getTimeInMillis();
this.rotateWithFileSize(tuple);
}
} catch (IOException var6) {
LOG.warn("write/sync failed.", var6);
this.collector.fail(tuple);
}
}
public void rotateWithFileSize(Tuple tuple) throws IOException {
syncHdfs(tuple);
this.collector.ack(tuple);
if (this.rotationPolicy.mark(tuple, this.offset)) {
this.rotateOutputFile();
this.offset = 0L;
this.rotationPolicy.reset();
}
}
public void rotateWithOutFileSize(Tuple tuple) throws IOException {
syncHdfs(tuple);
this.collector.ack(tuple);
this.rotateOutputFile();
this.offset = 0L;
this.rotationPolicy.reset();
}
public void syncHdfs(Tuple tuple) throws IOException {
byte[] e = this.format.format(tuple);
synchronized (this.writeLock) {
this.out.write(e);
this.offset += (long) e.length;
if (this.syncPolicy.mark(tuple, this.offset)) {
if (this.out instanceof HdfsDataOutputStream) {
((HdfsDataOutputStream) this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
} else {
this.out.hsync();
}
this.syncPolicy.reset();
}
}
}
public void closeOutputFile() throws IOException {
this.out.close();
}
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
LOG.info("Preparing HDFS Bolt...");
this.fs = FileSystem.get(URI.create(this.fsUrl), this.hdfsConfig);
this.tickTupleCount = 0;
this.normalTupleTime = 0;
this.tickTupleTime = 0;
}
public Path createOutputFile() throws IOException {
Path path = new Path(this.fileNameFormat.getPath(),
this.fileNameFormat.getName((long) this.rotation, System.currentTimeMillis()));
this.out = this.fs.create(path);
return path;
}
}
You can directly use this class in your project.
Thanks,

Hadoop: the Mapper didn't read files from multiple input paths

The Mapper didn't manage to read a file from multiple directories. Could anyone help?
I need to read one file in each mapper. I've added multiple input paths and implemented the custom WholeFileInputFormat, WholeFileRecordReader. In the map method, I don't need the input key. I make sure that each map can read a whole file.
Command line: hadoop jar AutoProduce.jar Autoproduce /input_a /input_b /output
I specified two input path----1.input_a; 2.input_b;
Run method snippets:
Job job = new Job(getConf());
job.setInputFormatClass(WholeFileInputFormat.class);
FileInputFormat.setInputPaths(job, new Path(args[0]), new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path(args[2]));
map method snippets:
public void map(NullWritable key, BytesWritable value, Context context){
FileSplit fileSplit = (FileSplit) context.getInputSplit();
System.out.println("Directory :" + fileSplit.getPath().toString());
......
}
Custom WholeFileInputFormat:
class WholeFileInputFormat extends FileInputFormat<NullWritable, BytesWritable> {
#Override
protected boolean isSplitable(JobContext context, Path file) {
return false;
}
#Override
public RecordReader<NullWritable, BytesWritable> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
WholeFileRecordReader reader = new WholeFileRecordReader();
reader.initialize(split, context);
return reader;
}
}
Custom WholeFileRecordReader:
class WholeFileRecordReader extends RecordReader<NullWritable, BytesWritable> {
private FileSplit fileSplit;
private Configuration conf;
private BytesWritable value = new BytesWritable();
private boolean processed = false;
#Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
this.fileSplit = (FileSplit) split;
this.conf = context.getConfiguration();
}
#Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (!processed) {
byte[] contents = new byte[(int) fileSplit.getLength()];
Path file = fileSplit.getPath();
FileSystem fs = file.getFileSystem(conf);
FSDataInputStream in = null;
try {
in = fs.open(file);
IOUtils.readFully(in, contents, 0, contents.length);
value.set(contents, 0, contents.length);
} finally {
IOUtils.closeStream(in);
}
processed = true;
return true;
}
return false;
}
#Override
public NullWritable getCurrentKey() throws IOException,InterruptedException {
return NullWritable.get();
}
#Override
public BytesWritable getCurrentValue() throws IOException,InterruptedException {
return value;
}
#Override
public float getProgress() throws IOException {
return processed ? 1.0f : 0.0f;
}
#Override
public void close() throws IOException {
// do nothing
}
}
PROBLEM:
After setting two input paths, all map tasks read files from only one directory..
Thanks in advance.
You'll have to use MultipleInputs instead of FileInputFormat in the driver. So your code should be as:
MultipleInputs.addInputPath(job, new Path(args[0]), <Input_Format_Class_1>);
MultipleInputs.addInputPath(job, new Path(args[1]), <Input_Format_Class_2>);
.
.
.
MultipleInputs.addInputPath(job, new Path(args[N-1]), <Input_Format_Class_N>);
So if you want to use WholeFileInputFormat for the first input path and TextInputFormat for the second input path, you'll have to use it the following way:
MultipleInputs.addInputPath(job, new Path(args[0]), WholeFileInputFormat.class);
MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class);
Hope this works for you!

Not getting value in midlet from servlet

I'm trying to send data from a midlet to a servlet and recieve back a response from the servlet but I don't get any value in response. I've tried to print it on command window and it seems to be null yet I expect only two values "ok" or "error". Please help me check the code and let me know what I should do to get the response right on the midlet. My code is below:
import javax.microedition.midlet.*;//midlet class package import
import javax.microedition.lcdui.*;//package for ui and commands
import javax.microedition.io.*;//
import java.io.*;
/**
* #author k'owino
*/
//Defining the midlet class
public class MvsMidlet extends MIDlet implements CommandListener {
private boolean midletPaused = false;//variable for paused state of midlet
//defining variables
private Display display;// Reference to Display object
private Form welForm;
private Form vCode;//vote code
private StringItem welStr;
private TextField phoneField;
private StringItem phoneError;
private Command quitCmd;
private Command contCmd;
//constructor of the midlet
public MvsMidlet() {
display = Display.getDisplay(this);//creating the display object
welForm = new Form("THE MVS");//instantiating welForm object
welStr = new StringItem("", "Welcome to the MVS, Busitema's mobile voter."
+ "Please enter the your phone number below");//instantiating welStr string item
phoneError = new StringItem("", "");//intantiating phone error string item
phoneField = new TextField("Phone number e.g 0789834141", "", 10, 3);//phone number field object
quitCmd = new Command("Quit", Command.EXIT, 0);//creating quit command object
contCmd = new Command("Continue", Command.OK, 0);//creating continue command object
welForm.append(welStr);//adding welcome string item to form
welForm.append(phoneField);//adding phone field to form
welForm.append(phoneError);//adding phone error string item to form
welForm.addCommand(contCmd);//adding continue command
welForm.addCommand(quitCmd);//adding quit command
welForm.setCommandListener(this);
display.setCurrent(welForm);
}
//start application method definition
public void startApp() {
}
//pause application method definition
public void pauseApp() {
}
//destroy application method definition
public void destroyApp(boolean unconditional) {
}
//Command action method definition
public void commandAction(Command c, Displayable d) {
if (d == welForm) {
if (c == quitCmd) {
exitMidlet();//call to exit midlet
} else {//if the command is contCmd
//place a waiting activity indicator
System.out.println("ken de man");
Thread t = new Thread() {
public void run() {
try {
//method to connect to server
sendPhone();
} catch (Exception e) {
}//end of catch
}//end of ru()
};//end of thread
t.start();//starting the thread
}//end of else
}//end of first if
}//end of command action
//defining method to exit midlet
public void exitMidlet() {
display.setCurrent(null);
destroyApp(true);
notifyDestroyed();
}//end of exitMidlet()
//defining sendPhone method
public void sendPhone() throws IOException {
System.out.println("ken de man");//check
HttpConnection http = null;//HttpConnection variable
OutputStream oStrm = null;//OutputStream variable
InputStream iStrm = null;//InputStream variable
String url = "http://localhost:8084/MvsWeb/CheckPhone";//server url
try {
http = (HttpConnection) Connector.open(url);//opening connection
System.out.println("connection made");//checking code
oStrm = http.openOutputStream();//opening output stream
http.setRequestMethod(HttpConnection.POST);//setting request type
http.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");//setting content type
byte data[] = ("phone=" + phoneField.getString()).getBytes();
oStrm.write(data);
iStrm = http.openInputStream();//openning input stream
if (http.getResponseCode() == HttpConnection.HTTP_OK) {
int length = (int) http.getLength();
String str;
if (length != -1) {
byte servletData[] = new byte[length];
iStrm.read(servletData);
str = new String(servletData);
} else // Length not available...
{
ByteArrayOutputStream bStrm = new ByteArrayOutputStream();
int ch;
while ((ch = iStrm.read()) != -1) {
bStrm.write(ch);
}
str = new String(bStrm.toByteArray());
bStrm.close();
}
System.out.println("de man");
System.out.println(str);
if (str.equals("ok")) {
//change to vcode form
} else {
//add error message to phone_error stingItem
}
}
} catch (Exception e) {
}
}
}//end of class definition
//servlet
import java.io.*;//package for io classes
import javax.servlet.ServletException;//package for servlet exception classes
import javax.servlet.http.*;
import java.sql.*;//package for sql classes
/**
*
* #author k'owino
*/
public class CheckPhone extends HttpServlet {
#Override
protected void doGet(HttpServletRequest req, HttpServletResponse res)
throws ServletException, IOException {
}
#Override
protected void doPost(HttpServletRequest req, HttpServletResponse res)
throws ServletException, IOException {
//declaring variables
PrintWriter pw;//PrintWriter object
String pnum;//phone parameter sent from client
Connection con = null;//connection variable
String dbDriver = "com.jdbc.mysql.Driver";//the database driver
String dbUrl = "jdbc:mysql://localhost/mvs_db";//the database url
String dbUser = "root";//database user
String dbPwd = "";//database password
PreparedStatement pstmt = null;//Prepared statement variable
String query = "select * from student where stud_phone = ?;";
ResultSet rs = null;//resultset variable
String dbPhone=null;//phone from database
//getting the "phone" parameter sent from client
pnum = req.getParameter("phone");//getting the "phone" parameter sent from client
System.out.println(pnum);
//setting the content type of the response
res.setContentType("text/html");
//creating a PrintWriter object
pw = res.getWriter();
pw.println("ken");
try{//trying to load the database driver and establish a connection to the database
Class.forName(dbDriver).newInstance();//loading the database driver
con = DriverManager.getConnection(dbUrl,dbUser,dbPwd);//establishing a connection to the database
System.out.println("connection established");
}
catch(Exception e){
}
//trying to query the database
try{
pstmt = con.prepareStatement(query);//preparing a statement
pstmt.setString(1, pnum);//setting the input parameter
rs = pstmt.executeQuery();//executing the query assigning to the resultset object
//extracring data from the resultset
while(rs.next()){
dbPhone = rs.getString("stud_phone");//getting the phone number from database
}//end of while
if(pnum.equals(dbPhone)){
pw.print("ok");
pw.close();
}
else{
pw.print("error");
pw.close();
}
}
catch(Exception e){
}
}
#Override
public String getServletInfo() {
return "Short description";
}// </editor-fold>
}

Resources