C#爬虫算法

c#数据采集

这段时间采集一些网络资源，考虑到采集源的防采集屏蔽，于是火车头采集器也排不上用场了，只能自己写一个，部分代码如下：

private void gotoRes() {

int num = 0; int dnum = 0;

string strMsg = \执行数据如下：\\r\\n\\r\\n\;

DataTable dt = Business.Business.GetUrlTable(Convert.ToInt32(starid.Text), Convert.ToInt32(endid.Text)); for (int i = 0; i < dt.Rows.Count; i++) {

if (Convert.ToInt32(dt.Rows[i][\]) == 0) {

string strPath = \; string strFilePath = \;

if (dt.Rows[i][\].ToString().Length > 10)

{

string[] resourceUrl = SplitString(dt.Rows[i][\].ToString().ToLower(), \);

for (int length = 5; length < resourceUrl.Length; length++)

{

strMsg += dt.Rows[i][\].ToString() + dt.Rows[i][\].ToString() + dt.Rows[i][\\].ToString() + \;

strPath += resourceUrl[length - 1];

strFilePath += strPath + \ + resourceUrl[length];

} }

if (!Directory.Exists(strPath)) {

Directory.CreateDirectory(strPath); }

if (downfile2(dt.Rows[i][\].ToString(), strFilePath)) {

if (Business.Business.UpDateRes(Convert.ToInt32(dt.Rows[i][\]))) {

strMsg += strFilePath + \下载完毕！\\r\\n\; dnum++; } } num++; }

msg.Text = strMsg + \共载入\ + num.ToString() + \条数据！下载数据：\ + dnum.ToString() + \。\; } }

public bool downfile2(string url, string LocalPath) { try {

Uri u = new Uri(url);

HttpWebRequest mRequest = (HttpWebRequest)WebRequest.Create(u);

mRequest.Method = \;

mRequest.ContentType = \oded\;

HttpWebResponse wr = (HttpWebResponse)mRequest.GetResponse();

statusBar1.Text = \开始下载文件

Stream sIn = wr.GetResponseStream();

FileStream fs = new FileStream(LocalPath, FileMode.Create, FileAccess.Write);

statusBar1.Text = \; long length = wr.ContentLength; int i = 0; long j = 0;

statusBar1.Text = \正在接收数据

\; \;

byte[] buffer = new byte[1024];

while ((i = sIn.Read(buffer, 0, buffer.Length)) > 0) {

j += i;

fs.Write(buffer, 0, i);

statusBar1.Text = \文件大小:\ + length.ToString() + \字节 当前下载:\ + j + \字节\; }

sIn.Close(); wr.Close();

statusBar1.Text = \文件下载完毕

文件大小\ + fs.Length.T

oString() + \字节\;

Console.WriteLine(\文件下载完毕

文件大小\ + fs.Length.T

oString() + \字节.\ + System.DateTime.Now.ToString()); fs.Close(); return true; }

catch { return false; } }