package vnet.com.weather1;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import vnet.com.update.Getdata;
/**
 * 正则方式抓取新浪天气新闻上的新闻
 * 地址http://weather.news.sina.com.cn/weather/news/index.html
 * @param args
 */
public class Newlist {
    private static final Log log = LogFactory.getLog(Newlist.class);
    /**
     * 测试
     * @param args
     */
    public  static void main(String args[]){
        Newlist n=new Newlist();
        String[] k=n.getNewList();
        for (int i=0;i<k.length;i++){
        System.out.println(k[i].replace("href=\"", "href=\"newinfo2.jsp?url="));
        }
        String[] m=n.getNewinfo("news/2008/1119/35261.html");
        for (int l=0;l<m.length;l++){        
            System.out.println(m[l]);    
        }
        
    }
    /**
     * 由url地址获得新闻内容string[]
     * 新闻中的图片下载到本地，文中新闻地址改成本地地址
     * @param url
     * @return
     */
    public String[] getNewinfo(String url){
        String URL="http://weather.news.sina.com.cn/"+url;
        //30是指取30段满足给出的正则条件的字符串，如果只找出10个，那数组后面的全为null
        String[] s = analysis("<p>(.*?)</p>" , getContent(URL) , 30);
        for (int i=0;i<s.length;i++){
            Pattern sp = Pattern.compile("src=\"(.*?)\"");
            Matcher matcher = sp.matcher(s[i]);
            if (matcher.find()){
                
                 String imageurl=analysis("src=\"(.*?)\"" , s[i] , 1)[0];
                 if(!imageurl.startsWith("http://")){
                     imageurl="http://weather.news.sina.com.cn/"+imageurl;
                  }
                System.out.println("新闻有图片:"+imageurl);
                String content=getContent(imageurl);
                  String[] images=imageurl.split("/");
                  String imagename=images[images.length-1];
                  System.out.println("图片名:"+imagename);
                  
                 
        try {
            File fwl = new File(imagename); 
            PrintWriter outl = new PrintWriter(fwl);
            outl.println(content);
            outl.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            System.out.println("s[i]:"+s[i]);
            //修改文件图片地址
            s[i]=s[i].replace(analysis("src=\"(.*?)\"" , s[i] , 1)[0], imagename);
            }
        }
        
        return s;
    }
    public  String[] getNewList(){
        String url="http://weather.news.sina.com.cn/weather/news/index.html";
        return getNewList(getContent(url));       
    }

    private  String[] getNewList(String content ){
        //String[] s = analysis("align=\"center\" valign=\"top\"><img src=\"../images/a(.*?).gif\" width=\"70\" height=\"65\"></td>" , content , 50);    
        String[] s = analysis("<li>(.*?)</li>" , content , 50);
        
        return s;
    }
    private String[] analysis(String pattern, String match , int i){
        Pattern sp = Pattern.compile(pattern);
        Matcher matcher = sp.matcher(match);
        String[] content = new String[i];
        for (int i1 = 0; matcher.find(); i1++){        
            content[i1] = matcher.group(1);       
        }
        //下面一段是为了剔除为空的串
        int l=0;
        for (int k=0;k<content.length;k++){
            if (content[k]==null){
                l=k;
                break;
            }
        }
        String[] content2;
        if (l!=0){
            content2=new String[l];
            for (int n=0;n<l;n++){
                content2[n]=content[n];
            }
             return content2;
        }else{
            return content;    
        }
       
    }
    /**
     * 由地址获取网页内容
     * @param strUrl
     * @return
    private String getContent(String strUrl){
        try{
            //URL url = new URL(strUrl);    
            //BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()));
            URLConnection uc = new URL(strUrl).openConnection(); 
               //通过修改http头的User-Agent来伪装成是通过浏览器提交的请求
              uc.setRequestProperty("User-Agent",  
                                    "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt)");                  
              System.out.println("-----------------------------------------");  
              System.out.println("Content-Length:     "+uc.getContentLength());  
              System.out.println("Set-Cookie:     "+uc.getHeaderField("Set-Cookie"));  
              System.out.println("-----------------------------------------"); 
              //获取文件头信息
              System.out.println("Header"+uc.getHeaderFields().toString());
              System.out.println("-----------------------------------------");  
            BufferedReader br=new BufferedReader(new InputStreamReader(uc.getInputStream(), "gb2312")); 
            String s = "";
            StringBuffer sb=new StringBuffer();
            while((s = br.readLine())!=null){
                sb.append(s+"\r\n");
            }
            System.out.println("长度+"+sb.toString().length());
            
            return sb.toString();
        }catch(Exception e){
            return "error open url" + strUrl;
        }
    }
    */

    public static  String getContent (String strUrl){
        URLConnection uc = null;
        String all_content=null;

       
    try {
               all_content =new  String();
               URL url = new URL(strUrl);

               uc = url.openConnection();
               uc.setRequestProperty("User-Agent",  
                                    "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP; DigExt)");                  
              System.out.println("-----------------------------------------");  
              System.out.println("Content-Length:     "+uc.getContentLength());  
              System.out.println("Set-Cookie:     "+uc.getHeaderField("Set-Cookie"));  
              System.out.println("-----------------------------------------"); 
              //获取文件头信息
              System.out.println("Header"+uc.getHeaderFields().toString());
              System.out.println("-----------------------------------------");  
               if (uc == null)
                   return null;

               InputStream ins = uc.getInputStream();
                ByteArrayOutputStream outputstream = new ByteArrayOutputStream();
               byte[] str_b = new byte[1024];
                   int i = -1;
                   while ((i=ins.read(str_b)) > 0) {
                    outputstream.write(str_b,0,i);
                   }
                   all_content = outputstream.toString();
                  // System.out.println(all_content);

           } catch (Exception e) {
               e.printStackTrace();
               log.error("获取网页内容出错");
           }finally{
               uc = null;
           }
          
          // return new String(all_content.getBytes("ISO8859-1"));
           System.out.println(all_content.length());
           return all_content;
       }
      
}