17.抓取用戶IT鐵人賽文章

2019 iT 邦幫忙鐵人賽

DAY 17

Modern Web

輕量高效.NET Core開源Blog引擎:Miniblog.Core系列第 17 篇

2019鐵人賽

暐翰

2018-10-17 22:24:12

12557 瀏覽

分享至

今天想做一個IT鐵人賽文章查詢程式，之後用來自動同步Blog避免手工重複複製。

建立Get讀取方法、讀取IT指定用戶鐵人賽文章

連結規則 : https://ithelp.ithome.com.tw/users/用戶ID/ironman/主題系列ID，如圖片
2018-10-17.22.08.24-image.png

為了不堵塞程序，使用async異步方式來讀取網頁連結獲取HTML

	public async Task<string> GetAsync(string uri)
	{
		HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
		request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

		using (HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync())
		using (Stream stream = response.GetResponseStream())
		using (StreamReader reader = new StreamReader(stream))
		{
			return await reader.ReadToEndAsync();
		}
	}

使用AngleSharp操作IT鐵人賽HTML的DOM結構。

使用方式很簡單，先宣告一個實體變數保存AngleSharp.HtmlParser，
HtmlParser Parse方法可以將HTML轉為DOM物件
使用QuerySelectorAll、QuerySelector以JQuery選擇器方式篩選DOM。
舉例:

private static readonly HtmlParser _parser = new HtmlParser();
var dom = _parser.Parse("<html></html>").QuerySelector("html");

抓取文章

首先觀察文章內容包含:發布日期、標題、內容、連結
可以使用class='profile-list__content'抓取文章DOM，如圖片
2018-10-17.21.34.47-image.png

抓取文章、連結、日期

使用AngleSharp的QuerySelector方法抓取class=qa-list__title下A TagDOM，DOM的text內容是文章標題，DOM的href屬性是連結。
2018-10-17.21.31.28-image.png
接著抓取文章DOM下的class=qa-list__info-time的title屬性，裡面存放發布時間。
2018-10-17.21.42.51-image.png

CODE:

var allpost = document.QuerySelectorAll(".profile-list__content");
foreach (var postInfo in allpost){
	var post = new Post();
	var titleAndLinkDom = postInfo.QuerySelector(".qa-list__title>a");
	post.Title = titleAndLinkDom.InnerHtml.Trim(); /*標題*/
	post.link = titleAndLinkDom.GetAttribute("href").Trim(); /*連結*/  
    post.PubDate = DateTime.Parse(postInfo.QuerySelector(".qa-list__info-time").GetAttribute("title").Trim());
}

文章內容

需要依靠剛剛抓取到的文章連結讀取HTML，抓取class=markdown__styleDOM的InnerHtml就是文章內容了。
2018-10-17.21.49.43-image.png

	private async Task<string> GetPostContentAsync(string posturl)
	{
		var htmlContent = (await GetAsync(posturl));
		var document = _parser.Parse(htmlContent);
		return document.QuerySelectorAll(".markdown__style").FirstOrDefault().InnerHtml;
	}

完整Code

public class ITIronManSyncPostService
{
	private static readonly HtmlParser _parser = new HtmlParser();
	public IList<Post> Posts { get; set; } = new List<Post>();
	private string _url { get; set; }
	
	public async static Task<ITIronManSyncPostService> GetITIronManPosts(string url)
	{
		var itironman = new ITIronManSyncPostService();
		itironman._url = url;
		await itironman.ExecuteAsync();
		return itironman;
	}

	private async Task ExecuteAsync()
	{
		//因為IT鐵人賽只需要三十篇文章，每頁10篇文章，抓取頁數取4頁就好
		for (int i = 1; i < 4; i++)
			await GetITIronManPostsAsync(_url + $"?page={i}");
	}

	private async Task GetITIronManPostsAsync(string url)
	{
		var htmlContent = (await GetAsync(url));
		var document = _parser.Parse(htmlContent);
		
		//獲取鐵人賽主題
		var article = document.QuerySelector(".qa-list__title--ironman");
		article.RemoveChild(article.QuerySelector("span"));/*移除系列文字*/
		var articleText = article.TextContent.Trim();
		
		//獲取鐵人賽:發布日期、標題、內容、連結
		var allpost = document.QuerySelectorAll(".profile-list__content");
		foreach (var postInfo in allpost)
		{
			var post = new Post();
			
			var titleAndLinkDom = postInfo.QuerySelector(".qa-list__title>a");
			post.Title = titleAndLinkDom.InnerHtml.Trim();
			post.link = titleAndLinkDom.GetAttribute("href").Trim();
			post.Content = GetPostContentAsync(post.link).Result.Trim();
			post.PubDate = DateTime.Parse(postInfo.QuerySelector(".qa-list__info-time").GetAttribute("title").Trim());
			post.Article = articleText;
			
			Posts.Add(post);
		}			
	}

	private async Task<string> GetPostContentAsync(string posturl)
	{
		var htmlContent = (await GetAsync(posturl));
		var document = _parser.Parse(htmlContent);
		return document.QuerySelectorAll(".markdown__style").FirstOrDefault().InnerHtml;
	}

	public async Task<string> GetAsync(string uri)
	{
		HttpWebRequest request = (HttpWebRequest)WebRequest.Create(uri);
		request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;

		using (HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync())
		using (Stream stream = response.GetResponseStream())
		using (StreamReader reader = new StreamReader(stream))
		{
			return await reader.ReadToEndAsync();
		}
	}

	public class Post
	{
		public string Title { get; set; }
		public string link { get; set; }
		public string Content { get; set; }
		public string Article { get; set; }
		public DateTime PubDate { get; set; }
	}
}