urk网络爬虫(Spider)Java实现原理

如果此

URL

没有被处 理过，它会添加到等待区，之后，它会

像其他

URL

相关的代码如下所示：

import .*;

import .*;

import .*;

import .*;

public class CheckLinks extends implements

Runnable,ISpiderReportable {

/**

* The constructor. Perform setup here.

*/

public CheckLinks()

{

//{{INIT_CONTROLS

setTitle(

getContentPane().setLayout(null);

setSize(405,288);

setVisible(true);

t(

getContentPane().add(label1);

nds(12,12,84,12);

t(

ionCommand(

getContentPane().add(begin);

nds(12,36,84,24);

getContentPane().add(url);

nds(108,36,288,24);

oscrolls(true);

izontalScrollBarPolicy(.

NTAL_SCROLLBAR_ALWAYS);

ticalScrollBarPolicy(.

AL_SCROLLBAR_ALWAYS);

que(true);

getContentPane().add(errorScroll);

nds(12,120,384,156);

table(false);

wport().add(errors);

nds(0,0,366,138);

t(

getContentPane().add(current);

nds(12,72,384,12);

t(

getContentPane().add(goodLinksLabel);

nds(12,96,192,12);

t(

getContentPane().add(badLinksLabel);

nds(216,96,96,12);

//}}

//{{INIT_MENUS

//}}

//{{REGISTER_LISTENERS

SymAction lSymAction = new SymAction();

ionListener(lSymAction);

//}}

}

/**

* Main method for the application

*

* @param args Not used

*/

static public void main(String args[])

{

(new CheckLinks()).setVisible(true);

}

/**

* Add notifications.

*/

public void addNotify()

{

// Record the size of the window prior to calling parent's

// addNotify.

Dimension size = getSize();

ify();

if ( frameSizeAdjusted )

return;

frameSizeAdjusted = true;

// Adjust size of frame according to the insets and menu bar

Insets insets = getInsets();

ar menuBar = getRootPane().getJMenuBar();

int menuBarHeight = 0;

if ( menuBar != null )

menuBarHeight = ferredSize().height;

setSize( + + , +

+ +

menuBarHeight);

}

// Used by addNotify

boolean frameSizeAdjusted = false;

//{{DECLARE_CONTROLS

label1 = new ();

/**

* The begin or cancel button

*/

n begin = new n();

/**

* The URL being processed

*/

ield url = new ield();

/**

* Scroll the errors.

*/

lPane errorScroll =

new lPane();

/**

* A place to store the errors created

*/

rea errors = new rea();

current = new ();

goodLinksLabel = new ();

badLinksLabel = new ();

//}}

//{{DECLARE_MENUS

//}}

/**

* The background spider thread

*/

protected Thread backgroundThread;

/**

* The spider object being used

*/

protected Spider spider;

/**

* The URL that the spider began with

*/

protected URL base;

/**

* How many bad links have been found

*/

protected int badLinksCount = 0;

/**

* How many good links have been found

*/

protected int goodLinksCount = 0;

/**

* Internal class used to dispatch events

*

* @author wuhailin

* @version 1.0

*/

class SymAction implements Listener {

public void actionPerformed(Event event)

{

Object object = rce();

if ( object == begin )

begin_actionPerformed(event);

}

}

/**

* Called when the begin or cancel buttons are clicked

*

* @param event The event associated with the button.

*/

void begin_actionPerformed(Event event)

{

if ( backgroundThread==null ) {

el(

backgroundThread = new Thread(this);

();

goodLinksCount=0;

badLinksCount=0;

} else {

();

}

}

/**

* Perform the background thread operation. This method

* actually starts the background thread.

*/

public void run()

{

try {

t(

spider = new Spider(this);

();

base = new URL(t());

(base);

();

Runnable doLater = new Runnable()

{

public void run()

{

t(

}

};

Later(doLater);

backgroundThread=null;

} catch ( MalformedURLException e ) {

UpdateErrors err = new UpdateErrors();

=

Later(err);

}

}

/**

* Called by the spider when a URL is found. It is here

* that links are validated.

*

* @param base The page that the link was found on.

* @param url The actual link address.

*/

public boolean spiderFoundURL(URL base,URL url)

{

UpdateCurrentStats cs = new UpdateCurrentStats();

= ng();

Later(cs);

if ( !checkLink(url) ) {

UpdateErrors err = new UpdateErrors();

= url+

Later(err);

badLinksCount++;