Socket

Reading an web page source using java

June 13, 2010 in Java

I needed to develop an web crawler few days ago. My first challange was to read a website source code using java. Here is the code I wrote to read a webpage. It will read the http response of url given and print it out.

package webcrawler; 

import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;  

/**  *  * @author Burhan  */

public class readUrl
{
    StringBuffer content;
    String contentLowerCase;
    String line;
    String type = null;
    URL url = null;
    URLConnection urlConnection;
    InputStream urlStream;
    DataInputStream html;

    public readUrl(String urlStr) throws MalformedURLException, IOException
    {
        url = new URL(urlStr);
        urlConnection = url.openConnection();
        urlStream = url.openStream();
        type = urlConnection.getContentType();
        if(type==null)
            return;
        else if( type.compareTo("text/html") != 0 ) // not allowed type
            return;             // only htmls are here
        content = new StringBuffer();

        html = new DataInputStream(urlStream);

        while ((line = html.readLine()) != null)
        {
            content.append(new StringBuffer(line));
            content.append('\n');
        }
        System.out.print(content);
    }

    public static void main(String args[])
    {
        try
        {
            new readUrl("http://www.dscripts.net");
        }
        catch(MalformedURLException ex)
        {
            ex.printStackTrace();
        }
        catch(IOException ex)
        {
            ex.printStackTrace();
        }
    }
}