1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

  public static void Main()

    {

        HttpClientDownloader downloader = new HttpClientDownloader();

        Core.Spider spider = Core.Spider.Create(new MyPageProcessor(), new QueueDuplicateRemovedScheduler()).AddPipeline(new MyPipeline()).SetThreadNum(1);

        var site = new Site() { EncodingName = "UTF-8" };

        for (int i = 1; i < 5; ++i)

        {

            site.AddStartUrl("http://www.youku.com/v_olist/c_97_g__a__sg__mt__lg__q__s_1_r_0_u_0_pt_0_av_0_ag_0_sg__pr__h__d_1_p_1.html");

        }

        spider.Site = site;

        spider.Start();

    }

    private class MyPipeline : IPipeline

    {

        public void Process(ResultItems resultItems, ISpider spider)

        {

            foreach (YoukuVideo entry in resultItems.Results["VideoResult"])

            {

                Console.WriteLine($"{entry.Name}:{entry.Click}");

            }

            //May be you want to save to database

            // 

        }

        public void Dispose()

        {

        }

    }

    private class MyPageProcessor : IPageProcessor

    {

        public void Process(Page page)

        {

            var totalVideoElements = page.Selectable.SelectList(Selectors.XPath("//div[@class='yk-col3']")).Nodes();

            List<YoukuVideo> results = new List<YoukuVideo>();

            foreach (var videoElement in totalVideoElements)

            {

                var video = new YoukuVideo();

                video.Name = videoElement.Select(Selectors.XPath("/div[4]/div[1]/a")).Value;

                video.Click = int.Parse(videoElement.Select(Selectors.Css("p-num")).Value.ToString());

                results.Add(video);

            }

            page.AddResultItem("VideoResult", results);

        }

        public Site Site => new Site { SleepTime = 0 };

    }

    public class YoukuVideo

    {

        public string Name { get; set; }

        public string Click { get; set; }

    }

1

2

redisServer:your redis server 

redisPassword:your redis password

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

public class JdSkuSpider : ISpiderContext

{

    public SpiderContextBuilder GetBuilder()

    {

        Log.TaskId = "JD SKU Weekly";

        SpiderContext context = new SpiderContext

        {

            SpiderName = "JD SKU " + DateTimeUtils.MONDAY_RUN_ID,

            CachedSize = 1,

            ThreadNum = 8,

            Site = new Site

            {

                EncodingName = "UTF-8"

            },

            Scheduler = new RedisScheduler()

            {

                Host = "redis",

                Port = 6379,

                Password = ""

            },

            StartUrls=new Dictionary<string, Dictionary<string, object>> {

                { "http://list.jd.com/list.html?cat=9987,653,655&page=1&go=0&JL=6_0_0&ms=5", new Dictionary<string, object> { { "name","手机" }, { "cat3","9987" } } },

            },

            Pipeline = new MysqlPipeline()

            {

                ConnectString = ""

            },

            Downloader = new HttpDownloader()

        };

        return new SpiderContextBuilder(context, typeof(Product));

    }

    [Schema("jd", "sku_v2", Suffix = TableSuffix.Monday)]

    [TargetUrl(new[] { @"page=[0-9]+" }, "//*[@id=\"J_bottomPage\"]")]

    [TypeExtractBy(Expression = "//div[contains(@class,'j-sku-item')]", Multi = true)]

    [Indexes(Primary = "sku")]

    public class Product : ISpiderEntity

    {

        private static readonly DateTime runId;

        static Product()

        {

            DateTime dt = DateTime.Now;

            runId = new DateTime(dt.Year, dt.Month, 1);

        }

        [StoredAs("category", DataType.String, 20)]

        [PropertyExtractBy(Expression = "name", Type = ExtractType.Enviroment)]

        public string CategoryName { get; set; }

        [StoredAs("cat3", DataType.String, 20)]

        [PropertyExtractBy(Expression = "cat3", Type = ExtractType.Enviroment)]

        public int CategoryId { get; set; }

        [StoredAs("url", DataType.Text)]

        [PropertyExtractBy(Expression = "./div[1]/a/@href")]

        public string Url { get; set; }

        [StoredAs("sku", DataType.String, 25)]

        [PropertyExtractBy(Expression = "./@data-sku")]

        public string Sku { get; set; }

        [StoredAs("commentscount", DataType.String, 20)]

        [PropertyExtractBy(Expression = "./div[@class='p-commit']/strong/a")]

        public long CommentsCount { get; set; }

        [StoredAs("shopname", DataType.String, 100)]

        [PropertyExtractBy(Expression = "./div[@class='p-shop hide']/span[1]/a[1]")]

        public string ShopName { get; set; }

        [StoredAs("name", DataType.String, 50)]

        [PropertyExtractBy(Expression = "./div[@class='p-name']/a/em")]

        public string Name { get; set; }

        [StoredAs("shopid", DataType.String, 25)]

        public string ShopId { get; set; }

        [StoredAs("venderid", DataType.String, 25)]

        [PropertyExtractBy(Expression = "./@venderid")]

        public string VenderId { get; set; }

        [StoredAs("jdzy_shop_id", DataType.String, 25)]

        [PropertyExtractBy(Expression = "./@jdzy_shop_id")]

        public string JdzyShopId { get; set; }

        [StoredAs("cdate", DataType.Time)]

        [PropertyExtractBy(Expression = "now", Type = ExtractType.Enviroment)]

        public DateTime CDate => DateTime.Now;

    }

}