正则表达式学习 ---- 抓取远程网页并解析HTML

 学习java的正则表达式，抓取网页并解析HTML部分内容   

 

 

import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;

public class HttpClientDemo {
	
	/**
	 * 
	 * @param url
	 * @return
	 * @throws Exception
	 */
	public static String getHTML(String url) throws Exception {
		HttpClient httpClient = new HttpClient();
		GetMethod getMethod = new GetMethod(url);
		int statusCode = httpClient.executeMethod(getMethod);
		if (statusCode != HttpStatus.SC_OK) {
			System.err.println("Method failed: " + getMethod.getStatusLine());
			return null;
		}
		// 读取内容
		byte[] responseBody = getMethod.getResponseBody();
		getMethod.releaseConnection();
		return new String(responseBody);

	}
	/**
	 * 
	 * @throws Exception
	 */
	public static void test(String url) throws Exception{
		
		String html = getHTML(url);
		Pattern p = null;
		Matcher m = null;
		StringBuffer sb0 = new StringBuffer();
		// ul正则
		String regex = "<ul class=\"d2_9\">([\\s\\S]*<li>)<a.*href='(.*)'.*>(.+?)</a> \\[(.*)\\]</li>([\\s].*)";
		// 链接正则
		String regexa = "<a.*href='(.*)'.*>(.+?)</a> \\[(.*)\\]";
		p = Pattern.compile(regex);
		// m = p.matcher(sb.toString());
		m = p.matcher(html);
		int count = 0;
		// ul字符串
		while (m.find()) {
			sb0.append(m.group());
		}
		//System.out.println(sb0.toString());
		p = Pattern.compile(regexa);
		m = p.matcher(sb0.toString());
		// 链接地址和标题
		while (m.find()) {
			System.out.println("地址:" + m.group(1));
			System.out.println("标题:" + m.group(2));
			System.out.println("时间:" + m.group(3));
			count++;
		}
		
		System.out.println("抓取条数："+count);

	}
	
	public static void main(String[] args) throws Exception {
		String url = "http://cpc.people.com.cn/GB/194302/194306/index.html";
		test(url);
		
	}
}

运行结果：

 

评论

1 楼 heymaomao 2012-06-11  

为什么我抓下来一堆乱码？