tests.t7 source code

1 /+ dub.sdl:
2     name "t7"
3     dflags "-I../source"
4     #dflags "-debug"
5     lflags "-lcares"
6     buildRequirements "allowWarnings"
7     dependency "hio" version="*"
8     dependency "nbuff" version="*"
9     debugVersions "hiossl"
10 +/
11 
12 module tests.t7;
13 
14 import std.experimental.logger;
15 import std.datetime;
16 import std.socket;
17 import std.format;
18 
19 import hio.scheduler;
20 import hio.tls;
21 import hio.events;
22 import hio.loop;
23 
24 import nbuff: Nbuff;
25 
26 static string[] hosts = [
27     "www.reuters.com",
28     "www.bloomberg.org",
29     "www.wsj.com",
30     "stumbleupon.com",
31     "www-media.stanford.edu",
32     "www.deviantart.com",
33     "www.wiley.com",
34     "themeforest.net",
35     "mashable.com",
36     "www.trustpilot.com",
37     "www.a8.net",
38     "www.uol.com.br",
39     "www.domraider.com",
40     "us.sagepub.com",
41     "hbr.org",
42     "static-service.prweb.com",
43     "www.usgs.gov",
44     "www.archives.gov",
45     "www.usc.edu",
46     "www.usa.gov",
47     "www.istockphoto.com",
48     "www.snapchat.com",
49     "www2.gotomeeting.com",
50     "bitbucket-marketing-cdn.atlassian.com",
51     "cdn-1.wp.nginx.com",
52     "www.worldbank.org",
53     "www.mlbstatic.com"
54 ];
55 
56 // enum host = "www.humblebundle.com";
57 enum port = 443;
58 // enum host = "127.0.0.1";
59 // enum port = 4433;
60 
61 void scrape(string host)
62 {
63     globalLogLevel = LogLevel.info;
64     IORequest iorq;
65     AsyncSSLSocket s;
66     ulong bytes;
67     hlEvLoop loop = getDefaultLoop();
68 
69     void io_callback(ref IOResult r) @safe
70     {
71         bytes += r.input.length;
72         if (r.timedout || r.error)
73         {
74             infof("done, %d bytes", bytes);
75             loop.stop();
76             s.close();
77             return;
78         }
79         iorq.to_read = 16*1024;
80         iorq.output = Nbuff();
81         s.io(loop, iorq, 5.seconds);
82     }
83     void connected(AppEvent ev) @safe
84     {
85         infof("connected result: %s", ev);
86         if ( ev != AppEvent.OUT)
87         {
88             info("=== Failed to connect ===");
89             loop.stop();
90             s.close();
91             return;
92         }
93         infof("=== CONNECTED to %s ===", host);
94         iorq.callback = &io_callback;
95         iorq.to_read = 1024;
96         iorq.output = Nbuff("GET / HTTP/1.1\nConnection: close\n");
97         iorq.output.append("Host: %s\n".format(host));
98         iorq.output.append("\n");
99         s.io(loop, iorq, 5.seconds);
100     }
101     InternetAddress addr;
102     try
103     {
104         addr = new InternetAddress(host, port);
105     }
106     catch(std.socket.AddressException e)
107     {
108         infof("socket exception %s", e);
109         return;
110     }
111     s = new AsyncSSLSocket();
112     s.open();
113     s.set_host(host);
114     s.connect(addr, loop, &connected, 1.seconds);
115     loop.run(10.seconds);
116     s.close();
117 }
118 void main()
119 {
120     foreach (host; hosts)
121     {
122         scrape(host);
123     }
124 }