LWP FAQ (1/1)

[Paul Kulchenko]

====================================================================

LWP FAQ by Paul Kulchenko (paulc...@yahoo.com), updated 14/04/2000

News:
fido7.ru.cgi.perl

Internet:
http://perl.ti.cz/LWP-FAQ.txt (Andrew Zhilenko)

====================================================================

1.1. How to get text file (http, ftp)?
1.2. How to get jpeg/gif/bmp file and return it?
1.3. How to access password protected file?
1.4. How to set up REFERER and other HTTP header parameters?
1.5. How to get specified part of file (first MAXSIZE bytes)?
1.6. How to get and set up cookies?
1.7. How to specify proxy servers?
1.8. How to check for redirect?
1.9. How to access URL with POST method?
1.10. How to upload file?

====================================================================

1.1. How to get text file (http, ftp)?

use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://www.yahoo.com/';
my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);
print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.2. How to get jpeg/gif/bmp file and return it?

use LWP::UserAgent;
use CGI qw(header -no_debug);
$URL =
'http://a100.g.akamaitech.net/7/100/70/0001/www.fool.com/art/new/butts/go99.
gif';
my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);
binmode(STDOUT);
print $res->is_success ? (header('image/gif'), $res->content)
: (header('text/html'), $res->status_line);

====================================================================

1.3. How to access password protected file?

BEGIN {
package RequestAgent;
use LWP::UserAgent;
@ISA = qw(LWP::UserAgent);

sub new { LWP::UserAgent::new(@_); }
sub get_basic_credentials { return 'user', 'password' }
}
use CGI qw(header -no_debug);

my $res = RequestAgent->new->request(new HTTP::Request GET => $URL);
print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.4. How to set up REFERER and other HTTP header parameters?

use LWP::UserAgent;
use HTTP::Headers;
use CGI qw(header -no_debug);

my $URL = 'http://localhost/cgi-bin/hello.cgi';
my $res = LWP::UserAgent->new->request(
new HTTP::Request(
GET => $URL,
new HTTP::Headers referer => 'http://www.yahoo.com'),
);
print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.5. How to get specified part of file (first MAXSIZE bytes)?
(Range added as suggested by Andrey Sapozhnikov and insisted by Artem
Chuprina)

use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://www.yahoo.com/';
my $MAXSIZE = 1024;
my $ENDRANGE = $MAXSIZE-1;

print header;
my $res = LWP::UserAgent->new->request(
new HTTP::Request(
GET => $URL,
new HTTP::Headers Range => "bytes 0-$ENDRANGE"), \&callback, $MAXSIZE);

sub callback { my($data, $response, $protocol) = @_; print $data; die }

====================================================================

1.6. How to get and set up cookies?

use LWP::UserAgent;
use CGI qw(header -no_debug);
use HTTP::Cookies;

my $URL = 'http://mail.yahoo.com/';

my $ua = new LWP::UserAgent;
my $res = $ua->request(new HTTP::Request GET => $URL);
my $cookie_jar = new HTTP::Cookies;
$cookie_jar->extract_cookies($res);

print header;
if ($res->is_success) {
my $req = new HTTP::Request GET => $URL;
$cookie_jar->add_cookie_header($req);
$res = $ua->request($req);
print $res->is_success ? $res->as_string : $res->status_line;
} else {
print $res->status_line;
}

====================================================================

1.7. How to specify proxy servers?

use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://www.yahoo.com/';
my $ua = new LWP::UserAgent;

$ua->proxy(['http', 'ftp'], 'http://proxy.sn.no:8001/');
$ua->proxy('gopher', 'http://proxy.sn.no:8001/');

my $res = $ua->request(new HTTP::Request GET => $URL);
print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.8. How to check for redirect?

use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://www.yahoo.com/';
my $res = LWP::UserAgent->new->request(new HTTP::Request GET => $URL);

print header;
print $res->request->url if $res->previous->is_redirect;

====================================================================

1.9. How to access URL with POST method?
(simplified by Andrey Babkin)

use HTTP::Request::Common;
use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://mail.yahoo.com/';
my $request = POST $URL

# default Content-Type: application/x-www-form-urlencoded
login => 'mylogin',
password => 'mypassword',
];

my $res = LWP::UserAgent->new->request($request);

print header, $res->is_success ? $res->content : $res->status_line;

====================================================================

1.10. How to upload file?

use HTTP::Request::Common;
use LWP::UserAgent;
use CGI qw(header -no_debug);

my $URL = 'http://localhost/cgi-bin/survey.cgi';
my $request = POST $URL,
Content_Type => 'form-data',
Content =>

name => 'Paul Kulchenko',
email => 'paulc...@yahoo.com',
surveyfile => ['./survey.dat'], # this file will be uploaded
];

my $res = LWP::UserAgent->new->request($request);

print header, $res->is_success ? $res->content : $res->status_line;

====================================================================