import java.io.File;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Hashtable;
public class GetWeb {
private int webDepth = 2;// 爬虫深度
private int intThreadNum = 10;// 线程数
private String strHomePage = "";// 主页地址
private String myDomain;// 域名
private String fPath = "web";// 储存网页文件的目录名
private ArrayList<String> arrUrls = new ArrayList<String>();// 存储未处理URL
private ArrayList<String> arrUrl = new ArrayList<String>();// 存储所有URL供建立索引
private Hashtable<String, Integer> allUrls = new Hashtable<String, Integer>();// 存储所有URL的网页号
private Hashtable<String, Integer> deepUrls = new Hashtable<String, Integer>();// 存储所有URL深度
private int intWebIndex = 0;// 网页对应文件下标，从0开始
private String charset = "GB2312";
private String report = "";
private long startTime;
private int webSuccessed = 0;
private int webFailed = 0;
public GetWeb(String s) {
this.strHomePage = s;
}
public GetWeb(String s, int i) {
this.strHomePage = s;
this.webDepth = i;
}
public synchronized void addWebSuccessed() {
webSuccessed++;
}
public synchronized void addWebFailed() {
webFailed++;
} |