1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
#!/usr/bin/ruby1.8
require 'tempfile'
#
# This is a class which allows a remote HTTP/HTTPS page to be downloaded
# it allows both the content and the HTTP status-code to be retrieved assuming
# a success was made.
#
# This code is *horrificly* bad, but required because net/http doesn't honour
# timouts under certain circumstances. I'm not proud of this code.
#
# Steve
# --
#
class WebFetch
#
# The URL & timeout period (in seconds) we were given in the constructor
#
attr_reader :url, :timeout
#
# The HTTP status code, and content, we received from fetching the URL
#
attr_reader :status, :text
#
# An error to return to the caller, on failure
#
attr_reader :error
#
# Constructor
#
def initialize( url, timeout = 10 )
@url = url
@timeout = timeout
# defaults
@status = -1
@error = ""
@text = ""
end
#
# Perform the fetch of the remote URL. Return "true" on success.
#
def fetch
#
# Generate a temporary file to contain the header from the server.
#
tmp_head = Tempfile.new('curl-header')
head = tmp_head.path
#
# Generate a temporary file to contain the body from the server.
#
tmp_body = Tempfile.new('curl-body')
body = tmp_body.path
#
# Shell out to curl (!!!) to do the fetch.
#
# Avoid using the actual shell to avoid a security risk
#
system( "curl",
"--max-time",
timeout.to_s,
"--silent",
"--location",
"--insecure",
"--dump-header",
head,
"--out",
body,
"--silent",
@url )
#
# If the header was empty then we're a failure.
#
# (A body might be legitimately empty.)
#
if ( File.size( head ) == 0 )
#
# Cleanup
#
File.unlink( body ) if ( File.exists?( body ) )
File.unlink( head ) if ( File.exists?( head ) )
#
# Store the error.
#
@error = "Fetch of #{@url} failed"
return false
end
#
# Get the HTTP status code, by parsing the HTTP headers.
#
# NOTE: We will replace the code with later ones - this gives
# the status code *after* any potential redirection(s) have
# completed.
#
File.open( head, "r").each_line do |line|
if ( line =~ /HTTP\/[0-9]\.[0-9]\s+([0-9]+)\s+/ )
@status = $1.dup
end
end
#
# Get the body from the server, by parsing the temporary file.
#
File.open( body, "r").each_line do |line|
@text << line
end
#
# Cleanup. We're done.
#
File.unlink( body ) if ( File.exists?( body ) )
File.unlink( head ) if ( File.exists?( head ) )
return true
end
#
# Return the HTTP status code the server responded with, if the fetch was successful.
#
def status
@status
end
#
# Return the HTTP content the server responded with, if the fetch was successful.
#
def content
@text
end
#
# Return the error, if the fetch failed.
#
def error
@error
end
end
|
