1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
# 准备一个建立http请求的基础函数
// NewHTTPRequest ...
func NewHTTPRequest(method string, uri string, header map[string]string, body io.Reader) (req *http.Request, err error) {
req, err = http.NewRequest(method, uri, body)
if err != nil {
return
}
//deafult
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("Accept-Language", "zh,zh-CN;q=0.9")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Connection", "keep-alive")
req.Header.Set("Pragma", "no-cache")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("UA", "patech-dialing")
req.Header.Set("User-Agent", "/patech-dialing/Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36")
if header == nil {
return
}
for k, v := range header {
// Set Header Host
if strings.EqualFold(k, "host") {
req.Host = v
continue
}
req.Header.Set(k, v)
}
return
}
# 定义用于记录时间节点的变量
var (
dnsStartTime time.Time
dnsDoneTime time.Time
connectStartTime time.Time
connectDoneTime time.Time
tlsHandshakeStartTime time.Time
tlsHandshakeDoneTime time.Time
gotConnTime time.Time
gotFirstResponseByteTime time.Time
requestStartTime time.Time
requestDoneTime time.Time
)
// 调用函数，建立http请求句柄
httpRequest, err := funcs.NewHTTPRequest(request.Method.String(), url.String(), request.Header, body)
if err != nil {
....
}
// 创建clienttrace，用于定制记录时间的需求
trace := &httptrace.ClientTrace{
DNSStart: func(_ httptrace.DNSStartInfo) {
dnsStartTime = time.Now()
},
DNSDone: func(info httptrace.DNSDoneInfo) {
dnsDoneTime = time.Now()
if info.Err != nil {
response.Common.StatusCode = common.StatusDNSResolveFail
logger.Warnf("httptrace dns err:%v", info.Err)
}
},
ConnectStart: func(_, _ string) {
connectStartTime = time.Now()
},
ConnectDone: func(_, addr string, err error) {
response.RemoteAddr = addr
connectDoneTime = time.Now()
if err != nil {
response.Common.StatusCode = common.StatusConnRefused
logger.Warnf("httptrace connect err:%v", err)
}
},
GotConn: func(_ httptrace.GotConnInfo) {
gotConnTime = time.Now()
},
GotFirstResponseByte: func() {
gotFirstResponseByteTime = time.Now()
},
TLSHandshakeStart: func() {
tlsHandshakeStartTime = time.Now()
},
TLSHandshakeDone: func(_ tls.ConnectionState, err error) {
tlsHandshakeDoneTime = time.Now()
if err != nil {
response.Common.StatusCode = common.StatusInsecureCert
logger.Warnf("httptrace tls err:%v", err)
}
},
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
httpRequest = httpRequest.WithContext(httptrace.WithClientTrace(ctx, trace))
//Transport
transport := funcs.NewTransport(httpRequest, request.Common.Ipv6, request.InsecureSkipVerify)
//Client
client := &http.Client{
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// always refuse to follow redirects, visit does that
// manually if required.
return http.ErrUseLastResponse
},
}
//do
requestStartTime = time.Now()
httpResponse, err := client.Do(httpRequest)
// 时间记录转为毫秒
// http trace time
requestDoneTime.Sub(requestStartTime).Nanoseconds()
if (!dnsDoneTime.IsZero()) && (!dnsDoneTime.IsZero()) {
response.DnsMsTime += int32(dnsDoneTime.Sub(dnsStartTime).Nanoseconds() / 1e6)
}
if (!connectDoneTime.IsZero()) && (!connectStartTime.IsZero()) {
response.TcpMsTime += int32(connectDoneTime.Sub(connectStartTime).Nanoseconds() / 1e6)
}
if (!tlsHandshakeDoneTime.IsZero()) && (!tlsHandshakeStartTime.IsZero()) {
response.SslMsTime += int32(tlsHandshakeDoneTime.Sub(tlsHandshakeStartTime).Nanoseconds() / 1e6)
}
if (!gotConnTime.IsZero()) && (!connectDoneTime.IsZero()) {
response.ClientMsTime += int32(gotConnTime.Sub(connectDoneTime).Nanoseconds() / 1e6)
}
if (!gotFirstResponseByteTime.IsZero()) && (!gotConnTime.IsZero()) {
response.FirstPackageMsTime += int32(gotFirstResponseByteTime.Sub(gotConnTime).Nanoseconds() / 1e6)
}
....
// 判断状态码
if response.Common.StatusCode >= 400 {
return
}
// 记录请求完成时间
requestDoneTime = time.Now()
if !gotFirstResponseByteTime.IsZero() {
response.DownloadMsTime += int32(requestDoneTime.Sub(gotFirstResponseByteTime).Nanoseconds() / 1e6)
}
//处理重定向
if funcs.IsRedirect(httpResponse) {
loc, err := httpResponse.Location()
if err != nil {
logger.Warnf("resp.Location err: %s", err)
response.Common.StatusCode = common.StatusRedirectsFail
return
}
response.RedirectsFollowed++
if response.RedirectsFollowed > int32(task.maxRedirects) {
logger.Warnf("MaxRedirects[%d]", response.RedirectsFollowed)
response.Common.StatusCode = common.StatusRedirectsFail
return
}
timeout = timeout - time.Now().Sub(requestStartTime)
response.RedirectMsTime = int32((time.Now().UnixNano() - response.Common.StartNsTimestamp) / 1e6)
execSinglePage(assignment, task, loc, request, response, timeout)
return
}
|
