c#数据采集
这段时间采集一些网络资源,考虑到采集源的防采集屏蔽,于是火车头采集器也排不上用场了,只能自己写一个,部分代码如下:
private void gotoRes() {
int num = 0; int dnum = 0;
string strMsg = \执行数据如下:\\r\\n\\r\\n\;
DataTable dt = Business.Business.GetUrlTable(Convert.ToInt32(starid.Text), Convert.ToInt32(endid.Text)); for (int i = 0; i < dt.Rows.Count; i++) {
if (Convert.ToInt32(dt.Rows[i][\]) == 0) {
string strPath = \; string strFilePath = \;
if (dt.Rows[i][\].ToString().Length > 10)
{
string[] resourceUrl = SplitString(dt.Rows[i][\].ToString().ToLower(), \);
for (int length = 5; length < resourceUrl.Length; length++)
{
strMsg += dt.Rows[i][\].ToString() + dt.Rows[i][\].ToString() + dt.Rows[i][\\].ToString() + \;
strPath += resourceUrl[length - 1];
strFilePath += strPath + \ + resourceUrl[length];
} }
if (!Directory.Exists(strPath)) {
Directory.CreateDirectory(strPath); }
if (downfile2(dt.Rows[i][\].ToString(), strFilePath)) {
if (Business.Business.UpDateRes(Convert.ToInt32(dt.Rows[i][\]))) {
strMsg += strFilePath + \下载完毕!\\r\\n\; dnum++; } } num++; }
msg.Text = strMsg + \共载入\ + num.ToString() + \条数据!下载数据:\ + dnum.ToString() + \。\; } }
public bool downfile2(string url, string LocalPath) { try {
Uri u = new Uri(url);
HttpWebRequest mRequest = (HttpWebRequest)WebRequest.Create(u);
mRequest.Method = \;
mRequest.ContentType = \oded\;
HttpWebResponse wr = (HttpWebResponse)mRequest.GetResponse();
statusBar1.Text = \开始下载文件
Stream sIn = wr.GetResponseStream();
FileStream fs = new FileStream(LocalPath, FileMode.Create, FileAccess.Write);
statusBar1.Text = \; long length = wr.ContentLength; int i = 0; long j = 0;
statusBar1.Text = \正在接收数据
\; \;
byte[] buffer = new byte[1024];
while ((i = sIn.Read(buffer, 0, buffer.Length)) > 0) {
j += i;
fs.Write(buffer, 0, i);
statusBar1.Text = \文件大小:\ + length.ToString() + \字节 当前下载:\ + j + \字节\; }
sIn.Close(); wr.Close();
statusBar1.Text = \文件下载完毕
文件大小\ + fs.Length.T
oString() + \字节\;
Console.WriteLine(\文件下载完毕
文件大小\ + fs.Length.T
oString() + \字节.\ + System.DateTime.Now.ToString()); fs.Close(); return true; }
catch { return false; } }
C#爬虫算法
![](/skin/haowen/images/icon_star.png)
![](/skin/haowen/images/icon_star.png)
![](/skin/haowen/images/icon_star.png)
![](/skin/haowen/images/icon_star.png)