艺海无涯,学无止境。欢迎一起探讨技术和人生,了解我

asp.net模拟浏览器抓取网页内容

      public static string GetHTML(string url)
        {
            ASCIIEncoding encoding = new ASCIIEncoding();
            HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(url);
            myRequest.Method = "GET";
            myRequest.ContentType = "text/html;";
            myRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:14.0) Gecko/20100101 Firefox/14.0.1";
            myRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            myRequest.KeepAlive = true;
            HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();
            StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
            string content = reader.ReadToEnd();
            return content;
        }
public string GetHTML2(string url) {
WebClient wc = new WebClient();
wc.Credentials = CredentialCache.DefaultCredentials;
byte[] btPageData = wc.DownloadData(url);
string strTargetHtml = Encoding.UTF8.GetString(btPageData);
wc.Dispose();
return strTargetHtml;
}