Hi,
I'm currently in the process of creating an application that uses a
large amount of data streamed from a web service. In this case, I'm
using The Guardian's Politics API to make an application about UK
General Elections. In some cases, I need to stream and parse about
750KB of data - one example of this is when I'm loading data for all
constituencies. (Example: http://www.guardian.co.uk/politics/api/general-election/1997/results/json)
When I'm parsing this data, I occasionally get an OutOfMemoryError
problem when creating the JSON object from the Buffer Reader -
admittedly this is happening a lot less often than when I was using
the streamToString function, but I still want to eliminate the
possibility of an OOM Error wherever I can.
So I'd like to ask, what's the best way of parsing large amounts of
JSON data streamed over HTTP into a single JSON object?
Here's the error log:
And here's RestHandler.java:
Any advice would be very much appreciated. Many thanks.
I'm currently in the process of creating an application that uses a
large amount of data streamed from a web service. In this case, I'm
using The Guardian's Politics API to make an application about UK
General Elections. In some cases, I need to stream and parse about
750KB of data - one example of this is when I'm loading data for all
constituencies. (Example: http://www.guardian.co.uk/politics/api/general-election/1997/results/json)
When I'm parsing this data, I occasionally get an OutOfMemoryError
problem when creating the JSON object from the Buffer Reader -
admittedly this is happening a lot less often than when I was using
the streamToString function, but I still want to eliminate the
possibility of an OOM Error wherever I can.
So I'd like to ask, what's the best way of parsing large amounts of
JSON data streamed over HTTP into a single JSON object?
Here's the error log:
Code:
[color=red]
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): Uncaught handler:
thread Thread-16 exiting due to uncaught exception
05-14 02:15:33.381: ERROR/AndroidRuntime(21862):
java.lang.OutOfMemoryError
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
java.lang.AbstractStringBuilder.enlargeBuffer(AbstractStringBuilder.java:
99)
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
java.lang.AbstractStringBuilder.append0(AbstractStringBuilder.java:
139)
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
java.lang.StringBuilder.append(StringBuilder.java:282)
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
java.io.BufferedReader.readLine(BufferedReader.java:415)
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
com.mdodd.ElectionMonitor.resources.RestHandler.connect(RestHandler.java:
189)
05-14 02:15:33.381: ERROR/AndroidRuntime(21862): at
com.mdodd.ElectionMonitor.resources.ConstituenciesThread.run(ConstituenciesThread.java:
49)
[/color]And here's RestHandler.java:
Code:
package com.mdodd.ElectionMonitor.resources;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.json.*;
/**
* Handles the REST interface provided by the web service.
*
* @author michael
*
*/
public class RestHandler {
URI url; // Address of the service
HttpClient hc; // The HTTP client used to manage the connection
int statusCode; // The HTTP status code of the response
JSONObject json;
// Constants
public static final String DELETE = "DELETE";
public static final String GET = "GET";
public static final String POST = "POST";
public static final String PUT = "PUT";
// Server constants
public static final String SERVER = "http://www.guardian.co.uk";
public static final String PORT = ":80";
public static final String RESOURCE_DIR = "/politics/api/general-
election/";
public static final int CONNECTION_TIMEOUT = 5000;
public static final int SOCKET_TIMEOUT = 15000;
public static final String REST_URI =
RestHandler.SERVER + RestHandler.RESOURCE_DIR;
/**
* Constructor
* @param url RESTful URL to connect to
*/
public RestHandler(String url)
{
try
{
this.url = new URI(url);
}
catch(URISyntaxException murl)
{
this.url = null;
}
this.statusCode = -1;
System.out.println("Creates REST resouce at " + this.url);
}
//
// GET and SET methods
//
/**
* Retrieves the JSON object populated from the connect() function
* @return The retrieved JSON object
*/
public JSONObject getJSON()
{
return json;
}
/**
* Get the HTTP status code of the connection
* @return The HTTP status code of the connection
*/
public int getStatusCode()
{
return this.statusCode;
}
/**
* Converts an inputStream into a readable string.
* @param in The InputStream to be converted
* @return The converted String
*/
/*
public String streamToString(InputStream in)
{
// Set up vars for reading
BufferedReader buffer = new BufferedReader(new
InputStreamReader(in));
StringBuilder sb = new StringBuilder();
String line;
try
{
//TODO - Getting an OutOfMemoryError problem when loading the 1997
election
while((line = buffer.readLine()) != null)
{
// Add the current line to the buffer
sb.append(line + "\n");
}
}
catch(OutOfMemoryError me)
{
System.err.println("Out of memory when downloading information");
me.printStackTrace();
}
catch(Exception e)
{
System.err.println("Error converting in function streamToString");
e.printStackTrace();
}
finally
{
// Close the stream
try
{
in.close();
}
catch(IOException e)
{
System.err.println("Error closing stream in function
streamToString");
e.printStackTrace();
}
}
return sb.toString();
}
*/
/**
* Establishes a connection to the requested service
*/
public void connect() throws JSONException
{
// Local vars
HttpResponse response;
HttpEntity entity;
// Set up the connection
HttpGet hGet = new HttpGet(url);
hGet.addHeader("Accept", "application/json");
// Set the timeouts
HttpParams hp = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(hp, 5000); // 5 seconds to
establish connection
HttpConnectionParams.setSoTimeout(hp, 10000); // 10 seconds to
transfer data
// Create the client
hc = new DefaultHttpClient(hp);
// try the connection
try
{
// Make the request
response = hc.execute(hGet);
entity = response.getEntity();
this.statusCode = response.getStatusLine().getStatusCode();
if(entity != null)
{
// Get the contents of the entity
InputStream in = entity.getContent();
/*
* Use JSON to parse
*
* The buffer is fed directly into the constructor
* as this prevents an OutOfMemoryError from
* occurring. Can't use a separate string!
*/
BufferedReader buffer = new BufferedReader(new
InputStreamReader(in));
json = new JSONObject(buffer.readLine());
}
}
catch(Exception e)
{
System.err.println("Unable to connect to REST
service due to " + e.getMessage());
e.printStackTrace();
}
}
/**
* Generic method for getting a particular attribute from a REST
service
* @param uri The RESTful URL to be accessed
* @param tag The attribute to be retrieved by the
* @return returns a URI object
*/
public static Object getFromURI(String uri, String tag)
{
// initial setup
RestHandler r = new RestHandler(uri);
Object result = new Object();
try
{
r.connect();
}
catch(JSONException je)
{
je.printStackTrace();
return null;
}
// Get the arrays
JSONObject tmp = r.getJSON();
try
{
// Try getting the object with that tag.
// Fails if tag not found
result = tmp.get(tag);
}
catch(JSONException joe)
{
// tag not found
result = null;
}
// Return our match, if there is one
return result;
}
/**
* Outputs the structure of the JSON object to System.out, for
debugging purposes
*/
public void printJSON()
{
try
{
System.out.print(json.toString(2));
}
catch(JSONException je)
{
System.out.println("Unable to print JSON layout due to " +
je.getMessage());
je.printStackTrace();
}
}
}Any advice would be very much appreciated. Many thanks.
