Download dynamic content from AJAX driven sites.

The task was to download the page content from a website that was driven by AJAX. So it was necessary to have a browser with JavaScript enabled and to wait a moment after download to give JavaScript/AJAX a chance to modify page content.

The SWT of Eclipse owns such a browser implementation. I used it in NetBeans - worked fine.

So here we go:

1st we've to import the swt libraries

import org.eclipse.swt.SWT;
import org.eclipse.swt.browser.Browser;
import org.eclipse.swt.browser.ProgressEvent;
import org.eclipse.swt.browser.ProgressListener;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;

Next we implement a progress listener. The time offset mentioned above is here in the completed method. Adapt it!

private class MyProgressListener implements ProgressListener {

private String content;
private Boolean done;

public MyProgressListener() {
this.done = false;

public String getContent() {
return content;

public void setDone(Boolean done) {
this.done = done;

public Boolean getDone() {
return done;

public void changed(ProgressEvent event) {
this.done = false;

private void completed(ProgressEvent event) {
Browser browser = (Browser) event.widget;
try {
//Here is where wait to let Javascript/Ajax stuff take place and modify the content
} catch (Exception ignored) {
content = browser.getText();
done = true;

Then we declare our myBrowser class. Adapt path in method initBrowser!

private class MyBrowser {

private Display display;
private Shell shell;
private Browser browser;
private MyProgressListener myProgressListener;

private void initBrowser() {
System.setProperty("swt.library.path", "/usr/lib/jni");

this.display = new Display(); = new Shell(display);
this.browser = new Browser(shell, SWT.NONE);

private String getURLContentByBrowser(String loadURL) {


this.myProgressListener = new MyProgressListener();

while (!myProgressListener.getDone() == true) {
if (!this.display.readAndDispatch()) {
return this.myProgressListener.getContent();

private void disposeBrowser() {;

And finally wi'll download the stuff

private String getURLContentByStream(String loadURL) {
String content = "";
try {
URL url = new URL(loadURL);
InputStream inputStream = url.openStream();
DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream));

String line = "";

while ((line = dataInputStream.readLine()) != null) {
content += line;


} catch (Exception e) {
return content;

Have phun

No comments: