/*
* lftp - file transfer program
*
* Copyright (c) 1996-2016 by Alexander V. Lukyanov (lav@yars.free.net)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include "trio.h"
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <errno.h>
#include <stdarg.h>
#include <time.h>
#include <limits.h>
#include <fnmatch.h>
#include <locale.h>
#include <assert.h>
#include "Http.h"
#include "ResMgr.h"
#include "log.h"
#include "url.h"
#include "HttpAuth.h"
#include "HttpDir.h"
#include "misc.h"
#include "buffer_ssl.h"
#include "buffer_zlib.h"
#include "ascii_ctype.h"
#if !HAVE_DECL_STRPTIME
CDECL char *strptime(const char *buf, const char *format, struct tm *tm);
#endif
#define super NetAccess
#define max_buf 0x10000
#define HTTP_DEFAULT_PORT "80"
#define HTTP_DEFAULT_PROXY_PORT "3128"
#define HTTPS_DEFAULT_PORT "443"
enum {
H_Continue=100,
H_Switching_Protocols=101,
H_Processing=102,
H_Ok=200,
H_Created=201,
H_Accepted=202,
H_Non_Authoritative_Information=203,
H_No_Content=204,
H_Reset_Content=205,
H_Partial_Content=206,
H_Multi_Status=207,
H_Moved_Permanently=301,
H_Found=302,
H_See_Other=303,
H_Not_Modified=304,
H_Use_Proxy=305,
H_Temporary_Redirect=307,
H_Bad_Request=400,
H_Unauthorized=401,
H_Payment_Required=402,
H_Forbidden=403,
H_Not_Found=404,
H_Method_Not_Allowed=405,
H_Not_Acceptable=406,
H_Proxy_Authentication_Required=407,
H_Request_Timeout=408,
H_Conflict=409,
H_Gone=410,
H_Length_Required=411,
H_Precondition_Failed=412,
H_Request_Entity_Too_Large=413,
H_Request_URI_Too_Long=414,
H_Unsupported_Media_Type=415,
H_Requested_Range_Not_Satisfiable=416,
H_Expectation_Failed=417,
H_Unprocessable_Entity=422,
H_Locked=423,
H_Failed_Dependency=424,
H_Too_Many_Requests=429,
H_Internal_Server_Error=500,
H_Not_Implemented=501,
H_Bad_Gateway=502,
H_Service_Unavailable=503,
H_Gateway_Timeout=504,
H_HTTP_Version_Not_Supported=505,
H_Insufficient_Storage=507,
};
/* Some status code validation macros: */
#define H_2XX(x) (((x) >= 200) && ((x) <= 299))
#define H_5XX(x) (((x) >= 500) && ((x) <= 599))
#define H_PARTIAL(x) ((x) == H_Partial_Content)
#define H_REDIRECTED(x) (((x) == H_Moved_Permanently) || ((x) == H_Found) || ((x) == H_See_Other) || ((x) == H_Temporary_Redirect))
#define H_EMPTY(x) (((x) == H_No_Content) || ((x) == H_Reset_Content))
#define H_CONTINUE(x) ((x) == H_Continue || (x) == H_Processing)
#define H_REQUESTED_RANGE_NOT_SATISFIABLE(x) ((x) == H_Requested_Range_Not_Satisfiable)
#define H_TRANSIENT(x) ((x)==H_Request_Timeout || (x)==H_Bad_Gateway || (x)==H_Service_Unavailable || (x)==H_Gateway_Timeout)
#define H_UNSUPPORTED(x) ((x)==H_Bad_Request || (x)==H_Not_Implemented)
#define H_AUTH_REQ(x) ((x)==H_Unauthorized || (x)==H_Proxy_Authentication_Required)
#ifndef EINPROGRESS
#define EINPROGRESS -1
#endif
enum { CHUNK_SIZE_UNKNOWN=-1 };
Http::Connection::Connection(int s,const char *c)
: closure(c), sock(s)
{
}
Http::Connection::~Connection()
{
close(sock);
/* make sure we free buffers before ssl */
recv_buf=0;
send_buf=0;
}
void Http::Connection::MakeBuffers()
{
send_buf=new IOBufferFDStream(
new FDStream(sock,"<output-socket>"),IOBuffer::PUT);
recv_buf=new IOBufferFDStream(
new FDStream(sock,"<input-socket>"),IOBuffer::GET);
}
void Http::Init()
{
state=DISCONNECTED;
tunnel_state=NO_TUNNEL;
body_size=-1;
bytes_received=0;
status_code=0;
status_consumed=0;
proto_version=0x10;
sent_eot=false;
last_method=0;
default_cwd="/";
keep_alive=false;
keep_alive_max=-1;
array_send=0;
chunked=false;
chunked_trailer=false;
chunk_size=CHUNK_SIZE_UNKNOWN;
chunk_pos=0;
request_pos=0;
no_ranges=false;
seen_ranges_bytes=false;
entity_date_set=false;
sending_proppatch=false;
no_cache_this=false;
no_cache=false;
auth_sent[0]=auth_sent[1]=0;
auth_scheme[0]=auth_scheme[1]=HttpAuth::NONE;
use_propfind_now=true;
retry_after=0;
hftp=false;
https=false;
use_head=true;
user_agent=0;
special=HTTP_NONE;
}
Http::Http() : super()
{
Init();
Reconfig();
}
Http::Http(const Http *f) : super(f)
{
Init();
Reconfig();
}
Http::~Http()
{
Close();
Disconnect();
}
void Http::MoveConnectionHere(Http *o)
{
conn=o->conn.borrow();
conn->ResumeInternal();
rate_limit=o->rate_limit.borrow();
last_method=o->last_method; o->last_method=0;
last_uri.move_here(o->last_uri);
last_url.move_here(o->last_url);
timeout_timer.Reset(o->timeout_timer);
state=CONNECTED;
tunnel_state=o->tunnel_state;
o->Disconnect();
ResumeInternal();
}
void Http::DisconnectLL()
{
Enter(this);
rate_limit=0;
if(conn)
{
LogNote(7,_("Closing HTTP connection"));
conn=0;
}
if(!Error() && !H_AUTH_REQ(status_code))
auth_sent[0]=auth_sent[1]=0;
if(state!=DONE && (real_pos>0 || special==HTTP_POST)
&& !Error() && !H_AUTH_REQ(status_code)) {
if(last_method && !strcmp(last_method,"POST"))
SetError(FATAL,_("POST method failed"));
else if(ModeIs(STORE))
SetError(STORE_FAILED,0);
}
if(ModeIs(STORE) && H_AUTH_REQ(status_code))
pos=real_pos=request_pos; // resend all the data again
last_method=0;
last_uri.unset();
last_url.unset();
ResetRequestData();
state=DISCONNECTED;
Leave(this);
}
void Http::ResetRequestData()
{
body_size=-1;
bytes_received=0;
real_pos=no_ranges?0:-1;
status.set(0);
status_consumed=0;
line.set(0);
sent_eot=false;
keep_alive=false;
keep_alive_max=-1;
array_send=fileset_for_info?fileset_for_info->curr_index():0;
chunked=false;
chunked_trailer=false;
chunk_size=CHUNK_SIZE_UNKNOWN;
chunk_pos=0;
request_pos=0;
propfind=0;
inflate=0;
seen_ranges_bytes=false;
entity_date_set=false;
}
void Http::Close()
{
if(mode==CLOSED)
return;
if(conn && conn->recv_buf)
conn->recv_buf->Roll(); // try to read any remaining data
if(conn && keep_alive && (keep_alive_max>0 || keep_alive_max==-1)
&& !ModeIs(STORE) && !conn->recv_buf->Eof() && (state==RECEIVING_BODY || state==DONE))
{
conn->recv_buf->Resume();
conn->recv_buf->Roll();
if(xstrcmp(last_method,"HEAD"))
{
// check if all data are in buffer
if(!chunked) // chunked is a bit complex, so don't handle it
{
bytes_received+=conn->recv_buf->Size();
conn->recv_buf->Skip(conn->recv_buf->Size());
}
if(!(body_size>=0 && bytes_received==body_size))
goto disconnect;
}
// can reuse the connection.
state=CONNECTED;
ResetRequestData();
rate_limit=0;
}
else
{
disconnect:
Disconnect();
DontSleep();
}
array_send=0;
no_cache_this=false;
auth_sent[0]=auth_sent[1]=0;
auth_scheme[0]=auth_scheme[1]=HttpAuth::NONE;
no_ranges=!QueryBool("use-range",hostname);
use_propfind_now=QueryBool("use-propfind",hostname);
special=HTTP_NONE;
special_data.set(0);
sending_proppatch=false;
super::Close();
}
void Http::Send(const xstring& str)
{
if(str.length()==0)
return;
LogSend(5,str);
conn->send_buf->Put(str);
}
void Http::Send(const char *format,...)
{
va_list va;
va_start(va,format);
xstring& str=xstring::vformat(format,va);
va_end(va);
Send(str);
}
void Http::Send(const HttpHeader *hdr)
{
Send("%s: %s\r\n",hdr->GetName(),hdr->GetValue());
}
void Http::AppendHostEncoded(xstring& buf,const char *host)
{
if(is_ipv6_address(host))
buf.append('[').append(host).append(']');
else
buf.append_url_encoded(host,URL_HOST_UNSAFE);
}
void Http::SendMethod(const char *method,const char *efile)
{
xstring& stripped_hostname=xstring::get_tmp(hostname);
stripped_hostname.truncate_at('%');
xstring ehost;
AppendHostEncoded(ehost,xidna_to_ascii(stripped_hostname));
if(portname) {
ehost.append(':');
ehost.append(url::encode(portname,URL_PORT_UNSAFE));
}
if(!use_head && !strcmp(method,"HEAD"))
method="GET";
last_method=method;
if(file_url)
{
efile=file_url;
if(!proxy)
efile+=url::path_index(efile);
else if(!strncmp(efile,"hftp://",7))
efile++;
}
if(hftp && mode!=LONG_LIST && mode!=CHANGE_DIR && mode!=MAKE_DIR
&& mode!=REMOVE && mode!=REMOVE_DIR
&& (strlen(efile)<7 || strncmp(efile+strlen(efile)-7,";type=",6))
&& QueryBool("use-type",hostname))
{
efile=xstring::format("%s;type=%c",efile,ascii?'a':'i');
}
/*
Handle the case when the user has not given us
get http://foobar.org (note the absense of the trailing /
It fixes segfault with a certain webserver which I've
seen ... (Geoffrey Lee <glee@gnupilgrims.org>).
*/
if(*efile=='\0')
efile="/";
last_uri.set(efile+(proxy?url::path_index(efile):0));
if(last_uri.length()==0)
last_uri.set("/");
if(proxy)
last_url.set(efile);
Send("%s %s HTTP/1.1\r\n",method,efile);
Send("Host: %s\r\n",ehost.get());
if(user_agent && user_agent[0])
Send("User-Agent: %s\r\n",user_agent);
if(!hftp)
{
const char *content_type=0;
if(!strcmp(method,"PUT"))
content_type=Query("put-content-type",hostname);
else if(!strcmp(method,"POST"))
content_type=Query("post-content-type",hostname);
if(content_type && content_type[0])
Send("Content-Type: %s\r\n",content_type);
const char *accept=Query("accept",hostname);
if(accept && accept[0])
Send("Accept: %s\r\n",accept);
accept=Query("accept-language",hostname);
if(accept && accept[0])
Send("Accept-Language: %s\r\n",accept);
accept=Query("accept-charset",hostname);
if(accept && accept[0])
Send("Accept-Charset: %s\r\n",accept);
accept=Query("accept-encoding",hostname);
if(accept && accept[0])
Send("Accept-Encoding: %s\r\n",accept);
const char *referer=Query("referer",hostname);
const char *slash="";
if(!xstrcmp(referer,"."))
{
referer=GetConnectURL(NO_USER+NO_PASSWORD);
if(last_char(referer)!='/' && !cwd.is_file)
slash="/";
}
if(referer && referer[0])
Send("Referer: %s%s\r\n",referer,slash);
xstring cookie;
MakeCookie(cookie,hostname,efile+(proxy?url::path_index(efile):0));
if(cookie.length()>0)
Send("Cookie: %s\r\n",cookie.get());
}
}
void Http::SendBasicAuth(const char *tag,const char *auth)
{
if(!auth || !*auth)
return;
int auth_len=strlen(auth);
char *buf64=string_alloca(base64_length(auth_len)+1);
base64_encode(auth,buf64,auth_len);
Send("%s: Basic %s\r\n",tag,buf64);
}
void Http::SendBasicAuth(const char *tag,const char *user,const char *pass)
{
/* Basic scheme */
SendBasicAuth(tag,xstring::cat(user,":",pass,NULL));
}
void Http::SendAuth(HttpAuth::target_t target,const char *user,const char *uri)
{
auth_scheme[target]=HttpAuth::NONE;
if(!user)
return;
HttpAuth *auth=HttpAuth::Get(target,GetFileURL(file,NO_USER),user);
if(auth && auth->Update(last_method,uri)) {
auth_sent[target]++;
Send(auth->GetHeader());
}
}
void Http::SendProxyAuth()
{
SendAuth(HttpAuth::PROXY,proxy_user,last_url);
}
void Http::SendAuth()
{
if(hftp && !auth_scheme[HttpAuth::WWW] && user && pass && QueryBool("use-authorization",proxy)) {
SendBasicAuth("Authorization",user,pass);
return;
}
SendAuth(HttpAuth::WWW,user?user:auth_user,last_uri);
}
void Http::SendCacheControl()
{
const char *cc_setting=Query("cache-control",hostname);
const char *cc_no_cache=(no_cache || no_cache_this)?"no-cache":0;
if(!*cc_setting)
cc_setting=0;
if(!cc_setting && !cc_no_cache)
return;
int cc_no_cache_len=xstrlen(cc_no_cache);
if(cc_no_cache && cc_setting)
{
const char *pos=strstr(cc_setting,cc_no_cache);
if(pos && (pos==cc_setting || pos[-1]==' ')
&& (pos[cc_no_cache_len]==0 || pos[cc_no_cache_len]==' '))
cc_no_cache=0, cc_no_cache_len=0;
}
xstring& cc=xstring::join(",",2,cc_no_cache,cc_setting);
if(*cc)
Send("Cache-Control: %s\r\n",cc.get());
}
bool Http::ModeSupported()
{
switch((open_mode)mode)
{
case CLOSED:
case QUOTE_CMD:
case LIST:
case CHANGE_MODE:
case LINK:
case SYMLINK:
return false;
case CONNECT_VERIFY:
case RETRIEVE:
case STORE:
case MAKE_DIR:
case CHANGE_DIR:
case ARRAY_INFO:
case REMOVE_DIR:
case REMOVE:
case LONG_LIST:
case RENAME:
return true;
case MP_LIST:
#if USE_EXPAT
return QueryBool("use-propfind",hostname);
#else
// without XML parser it is meaningless to retrieve XML file info.
return false;
#endif
}
abort(); // should not happen
}
void Http::DirFile(xstring& path,const xstring& ecwd,const xstring& efile) const
{
const int base=path.length();
if(efile[0]=='/') {
path.append(efile);
} else if(efile[0]=='~' || ecwd.length()==0 || (ecwd.eq("~") && !hftp)) {
path.append('/');
path.append(efile);
} else {
size_t min_len=path.length()+1;
if(ecwd[0]!='/')
path.append('/');
path.append(ecwd);
if(ecwd.last_char()!='/' && efile.length()>0)
path.append('/');
// reduce . and .. at beginning of efile:
// * get the minimum path length (so that we don't remove ~user)
// * skip .; handle .. using basename_ptr to chomp the path.
if(path[min_len]=='~') {
while(path[min_len] && path[min_len]!='/')
++min_len;
if(path[min_len]=='/')
++min_len;
}
const char *e=efile;
while(e[0]=='.') {
if(e[1]=='/' || e[1]==0)
++e;
else if(e[1]=='.' && (e[2]=='/' || e[2]==0)) {
if(path.length()<=min_len)
break;
const char *bn=basename_ptr(path+min_len);
path.truncate(bn-path);
e+=2;
} else
break;
if(*e=='/')
++e;
}
path.append(e);
}
// remove "/~" or "/~/"
if(path[base+1]=='~' && path[base+2]==0)
path.truncate(base+1);
else if(path[base+1]=='~' && path[base+2]=='/')
path.set_substr(base,2,"");
}
void Http::SendPropfind(const xstring& efile,int depth)
{
SendMethod("PROPFIND",efile);
Send("Depth: %d\r\n",depth);
if(allprop.length()>0)
{
Send("Content-Type: text/xml\r\n");
Send("Content-Length: %d\r\n",int(allprop.length()));
}
}
void Http::SendPropfindBody()
{
Send(allprop);
}
const xstring& Http::FormatLastModified(time_t lm)
{
static const char weekday_names[][4]={
"Sun","Mon","Tue","Wed","Thu","Fri","Sat"
};
const struct tm *t=gmtime(&lm);
return xstring::format("%s, %2d %s %04d %02d:%02d:%02d GMT",
weekday_names[t->tm_wday],t->tm_mday,month_names[t->tm_mon],
t->tm_year+1900,t->tm_hour,t->tm_min,t->tm_sec);
}
void Http::SendProppatch(const xstring& efile)
{
SendMethod("PROPPATCH",efile);
xstring prop(
"<?xml version=\"1.0\" ?>"
"<propertyupdate xmlns=\"DAV:\">"
"<set>"
"<prop>"
"<getlastmodified>");
prop.append(FormatLastModified(entity_date)).append(
"</getlastmodified>"
"</prop>"
"</set>"
"</propertyupdate>");
Send("Content-Type: text/xml\r\n");
Send("Content-Length: %d\r\n",int(prop.length()));
Send("\r\n");
Send(prop);
}
void Http::SendRequest(const char *connection,const char *f)
{
xstring efile;
xstring ecwd;
bool add_slash=true;
if(mode==CHANGE_DIR && new_cwd && new_cwd->url)
{
const char *efile_c=new_cwd->url+url::path_index(new_cwd->url);
if(!*efile_c)
efile_c="/";
efile.set(efile_c);
add_slash=false;
}
else
efile.set(url::encode(f,URL_PATH_UNSAFE));
if(cwd.url)
ecwd.set(cwd.url+url::path_index(cwd.url));
else
{
ecwd.set(url::encode(cwd,URL_PATH_UNSAFE));
if(hftp && ecwd[0]=='/' && ecwd[1]!='~')
{
// root directory in ftp urls needs special encoding. (/%2Fpath)
ecwd.set_substr(1,0,"%2F");
}
}
if(cwd.is_file)
{
if(efile[0])
ecwd.truncate(basename_ptr(ecwd+(!strncmp(ecwd,"/~",2)))-ecwd);
add_slash=false;
}
if(mode==CHANGE_DIR && new_cwd && !new_cwd->url)
add_slash=!new_cwd->is_file;
xstring pfile;
if(proxy && !https)
{
const char *proto="http";
if(hftp)
proto="ftp";
pfile.vset(proto,"://",NULL);
if(hftp && user && pass)
{
pfile.append(url::encode(user,URL_USER_UNSAFE));
if(!QueryBool("use-authorization",proxy))
{
pfile.append(':');
pfile.append(url::encode(pass,URL_PASS_UNSAFE));
}
pfile.append('@');
}
AppendHostEncoded(pfile,hostname);
if(portname)
{
pfile.append(':');
pfile.append(url::encode(portname,URL_PORT_UNSAFE));
}
}
else
{
pfile.set("");
}
DirFile(pfile,ecwd,efile);
efile.set(pfile);
if(pos==0)
real_pos=0;
if(ModeIs(STORE)) // can't seek before writing
real_pos=pos;
#ifdef DEBUG_MP_LIST
if(mode==RETRIEVE && file[0]==0)
mode=MP_LIST;
#endif
switch((open_mode)mode)
{
case CLOSED:
case CONNECT_VERIFY:
abort(); // cannot happen
case QUOTE_CMD:
switch(special)
{
case HTTP_POST:
entity_size=special_data.length();
goto send_post;
case HTTP_MOVE:
case HTTP_COPY:
SendMethod(special==HTTP_MOVE?"MOVE":"COPY",efile);
Send("Destination: %s\r\n",special_data.get());
break;
case HTTP_PROPFIND:
SendMethod("PROPFIND",efile);
Send("Depth: 1\r\n"); // directory listing required
break;
case HTTP_NONE:
abort(); // cannot happen
}
break;
case LIST:
case CHANGE_MODE:
case LINK:
case SYMLINK:
abort(); // unsupported
case RETRIEVE:
retrieve:
SendMethod("GET",efile);
if(pos>0 && !no_ranges)
{
if(limit==FILE_END)
Send("Range: bytes=%lld-\r\n",(long long)pos);
else
Send("Range: bytes=%lld-%lld\r\n",(long long)pos,(long long)limit-1);
}
break;
case STORE:
if(sending_proppatch) {
SendProppatch(efile);
break;
}
if(hftp || strcasecmp(Query("put-method",hostname),"POST"))
SendMethod("PUT",efile);
else
{
send_post:
SendMethod("POST",efile);
pos=0;
}
if(entity_size>=0)
Send("Content-length: %lld\r\n",(long long)(entity_size-pos));
if(pos>0 && entity_size<0)
{
request_pos=pos;
if(limit==FILE_END)
Send("Range: bytes=%lld-\r\n",(long long)pos);
else
Send("Range: bytes=%lld-%lld\r\n",(long long)pos,(long long)limit-1);
}
else if(pos>0)
{
request_pos=pos;
Send("Range: bytes=%lld-%lld/%lld\r\n",(long long)pos,
(long long)((limit==FILE_END || limit>entity_size ? entity_size : limit)-1),
(long long)entity_size);
}
if((entity_date!=NO_DATE) && (entity_date>0L && entity_date<INT_MAX))
{
char d[256];
strftime(d, sizeof(d), "%a, %d %b %H:%M:%S %Y GMT", gmtime(&entity_date));
Send("Last-Modified: %s\r\n",d);
Send("X-OC-MTime: %ld\r\n",(long)entity_date); // for OwnCloud
}
break;
case CHANGE_DIR:
case LONG_LIST:
case MP_LIST:
case MAKE_DIR:
if(last_char(efile)!='/' && add_slash)
efile.append('/');
if(mode==CHANGE_DIR)
{
if(use_propfind_now)
SendPropfind(efile,0);
else
SendMethod("HEAD",efile);
}
else if(mode==LONG_LIST)
goto retrieve;
else if(mode==MAKE_DIR)
{
if(QueryBool("use-mkcol"))
SendMethod("MKCOL",efile);
else
{
SendMethod("PUT",efile);
Send("Content-Length: 0\r\n");
}
pos=entity_size=0;
}
else if(mode==MP_LIST)
{
SendPropfind(efile,1);
pos=0;
}
break;
case(REMOVE):
SendMethod("DELETE",efile);
Send("Depth: 0\r\n"); // deny directory removal
break;
case(REMOVE_DIR):
if(efile.last_char()!='/')
efile.append('/');
SendMethod("DELETE",efile);
break;
case ARRAY_INFO:
if(use_propfind_now)
SendPropfind(efile,0);
else
SendMethod("HEAD",efile);
break;
case RENAME:
{
SendMethod("MOVE",efile);
Send("Destination: %s\r\n",GetFileURL(file1).get());
}
}
if(proxy && !https)
SendProxyAuth();
SendAuth();
if(no_cache || no_cache_this)
Send("Pragma: no-cache\r\n"); // for HTTP/1.0 compatibility
SendCacheControl();
if(mode==ARRAY_INFO && !use_head)
connection="close";
else if(!ModeIs(STORE))
connection="keep-alive";
if(mode!=ARRAY_INFO || connection)
Send("Connection: %s\r\n",connection?connection:"close");
Send("\r\n");
if(special==HTTP_POST)
{
if(special_data)
Send("%s",special_data.get());
entity_size=NO_SIZE;
}
else if(!xstrcmp(last_method,"PROPFIND"))
SendPropfindBody();
keep_alive=false;
chunked=false;
chunked_trailer=false;
chunk_size=CHUNK_SIZE_UNKNOWN;
chunk_pos=0;
request_pos=0;
inflate=0;
no_ranges=!QueryBool("use-range",hostname);
conn->send_buf->SetPos(0);
}
int Http::SendArrayInfoRequest()
{
// skip to next needed file
for(FileInfo *fi=fileset_for_info->curr(); fi; fi=fileset_for_info->next())
if(fi->need)
break;
if(array_send<fileset_for_info->curr_index())
array_send=fileset_for_info->curr_index();
if(state!=CONNECTED)
return 0;
int m=1;
if(keep_alive && use_head)
{
m=keep_alive_max;
if(m==-1)
m=100;
}
int req_count=0;
while(array_send-fileset_for_info->curr_index()<m
&& array_send<fileset_for_info->count())
{
FileInfo *fi=(*fileset_for_info)[array_send++];
if(fi->need==0)
continue;
xstring *name=&fi->name;
if(fi->filetype==fi->DIRECTORY && name->last_char()!='/') {
name=&xstring::get_tmp(*name);
name->append('/');
}
if(fi->uri)
file_url.set(dir_file(GetConnectURL(),fi->uri));
else
file_url.unset();
SendRequest(array_send==fileset_for_info->count()-1 ? 0 : "keep-alive", *name);
req_count++;
}
return req_count;
}
void Http::ProceedArrayInfo()
{
for(;;)
{
// skip to next needed file
FileInfo *fi=fileset_for_info->next();
if(!fi || fi->need)
break;
}
if(!fileset_for_info->curr())
{
LogNote(10,"that was the last file info");
// received all requested info.
state=DONE;
return;
}
// we can avoid reconnection if server supports it.
if(keep_alive && (keep_alive_max>1 || keep_alive_max==-1)
&& (use_head || use_propfind_now))
{
// we'll have to receive next header, unset the status
status.set(0);
status_code=0;
state=CONNECTED;
SendArrayInfoRequest();
state=RECEIVING_HEADER;
}
else
{
Disconnect();
DontSleep();
}
}
void Http::NewAuth(const char *hdr,HttpAuth::target_t target,const char *user,const char *pass)
{
if(!user || !pass)
return;
// FIXME: keep a request queue, get the URI from the queue.
const char *uri=GetFileURL(file,NO_USER);
Ref<HttpAuth::Challenge> chal(new HttpAuth::Challenge(hdr));
bool stale=chal->GetParam("stale").eq_nc("true");
if(auth_sent[target]>(stale?1:0))
return;
HttpAuth::scheme_t new_scheme=chal->GetSchemeCode();
if(new_scheme<=auth_scheme[target])
return;
if(HttpAuth::New(target,uri,chal.borrow(),user,pass))
auth_scheme[target]=new_scheme;
}
void Http::HandleHeaderLine(const char *name,const char *value)
{
// use a perfect hash
#define hh(L,C) ((L)+(C)*3)
#define hhc(S,C) hh(sizeof((S))-1,(C))
#define case_hh(S,C) case hhc((S),(C)): if(strcasecmp(name,(S))) break;
switch(hh(strlen(name),c_toupper(name[0]))) {
case_hh("Content-Length",'C') {
long long bs=0;
if(1!=sscanf(value,"%lld",&bs))
return;
if(bs<0) // try to workaround broken servers
bs+=0x100000000LL;
body_size=bs;
if(mode==ARRAY_INFO && H_2XX(status_code)
&& xstrcmp(last_method,"PROPFIND"))
{
FileInfo *fi=fileset_for_info->curr();
fi->SetSize(body_size);
TrySuccess();
}
return;
}
case_hh("Content-Range",'C') {
long long first,last,fsize;
if(H_REQUESTED_RANGE_NOT_SATISFIABLE(status_code))
{
if(sscanf(value,"%*[^/]/%lld",&fsize)!=1)
return;
if(opt_size)
*opt_size=fsize;
return;
}
if(sscanf(value,"%*s %lld-%lld/%lld",&first,&last,&fsize)!=3)
return;
real_pos=first;
if(last==-1)
last=fsize-first-1;
if(body_size<0)
body_size=last-first+1;
if(!ModeIs(STORE) && !ModeIs(MAKE_DIR))
entity_size=fsize;
if(opt_size && H_2XX(status_code))
*opt_size=fsize;
return;
}
case_hh("Last-Modified",'L') {
if(!H_2XX(status_code))
return;
time_t t=Http::atotm(value);
if(t==ATOTM_ERROR)
return;
if(opt_date)
*opt_date=t;
if(mode==ARRAY_INFO && !propfind)
{
FileInfo *fi=fileset_for_info->curr();
fi->SetDate(t,0);
TrySuccess();
}
return;
}
case_hh("Location",'L')
if(value[0]=='/' && value[1]=='/')
location.vset(GetProto(),":",value,NULL);
else if(value[0]=='/')
location.vset(GetConnectURL().get(),value,NULL);
else
location.set(value);
return;
case_hh("Retry-After",'R')
retry_after=0;
sscanf(value,"%ld",&retry_after);
return;
case_hh("Keep-Alive",'K') {
keep_alive=true;
const char *m=strstr(value,"max=");
if(m) {
if(sscanf(m+4,"%d",&keep_alive_max)!=1)
keep_alive=false;
} else
keep_alive_max=100;
return;
}
case_hh("Proxy-Connection",'P')
goto case_Connection;
case_hh("Connection",'C')
case_Connection:
if(!strcasecmp(value,"keep-alive"))
keep_alive=true;
else if(!strcasecmp(value,"close"))
keep_alive=false;
return;
case_hh("Transfer-Encoding",'T')
if(!strcasecmp(value,"identity"))
return;
if(!strcasecmp(value,"chunked"))
{
chunked=true;
chunked_trailer=false;
chunk_size=CHUNK_SIZE_UNKNOWN; // expecting first chunk
chunk_pos=0;
}
return;
case_hh("Content-Encoding",'C')
content_encoding.set(value);
return;
case_hh("Accept-Ranges",'A')
if(!strcasecmp(value,"none"))
no_ranges=true;
if(strstr(value,"bytes"))
seen_ranges_bytes=true;
return;
case_hh("Set-Cookie",'S')
if(!hftp && QueryBool("set-cookies",hostname))
SetCookie(value);
return;
case_hh("Content-Disposition",'C') {
const char *filename=strstr(value,"filename=");
if(!filename)
return;
filename=HttpHeader::extract_quoted_value(filename+9);
SetSuggestedFileName(filename);
return;
}
case_hh("Content-Type",'C') {
entity_content_type.set(value);
const char *cs=strstr(value,"charset=");
if(cs)
{
cs=HttpHeader::extract_quoted_value(cs+8);
entity_charset.set(cs);
}
return;
}
case_hh("WWW-Authenticate",'W') {
if(status_code!=H_Unauthorized)
return;
if(user && pass)
NewAuth(value,HttpAuth::WWW,user,pass);
else
NewAuth(value,HttpAuth::WWW,auth_user,auth_pass);
return;
}
case_hh("Proxy-Authenticate",'P') {
if(status_code!=H_Proxy_Authentication_Required)
return;
NewAuth(value,HttpAuth::PROXY,proxy_user,proxy_pass);
return;
}
case_hh("X-OC-MTime",'X') {
if(!strcasecmp(value,"accepted"))
entity_date_set=true;
}
default:
break;
}
LogNote(10,"unhandled header line `%s'",name);
}
static
const char *find_eol(const char *p,int len,int *eol_size)
{
*eol_size=1;
for(int i=0; i<len; i++,p++)
{
if(p[0]=='\n')
return p;
if(i+1<len && p[1]=='\n' && p[0]=='\r')
{
*eol_size=2;
return p;
}
}
*eol_size=0;
return 0;
}
void Http::GetBetterConnection(int level)
{
if(level==0)
return;
for(FA *fo=FirstSameSite(); fo!=0; fo=NextSameSite(fo))
{
Http *o=(Http*)fo; // we are sure it is Http.
if(!o->conn || o->state==CONNECTING)
continue;
if(o->tunnel_state==TUNNEL_WAITING)
continue;
if(o->state!=CONNECTED || o->mode!=CLOSED)
{
if(level<2)
continue;
if(!connection_takeover || (o->priority>=priority && !o->IsSuspended()))
continue;
o->Disconnect();
return;
}
// so borrow the connection
MoveConnectionHere(o);
return;
}
}
int Http::Do()
{
int m=STALL;
int res;
const char *error;
const char *buf;
int len;
// check if idle time exceeded
if(mode==CLOSED && conn && idle_timer.Stopped())
{
LogNote(1,_("Closing idle connection"));
Disconnect();
return m;
}
if(home.path==0)
set_home(default_cwd);
if(Error())
return m;
if(propfind)
{
if(propfind->Error())
{
propfind=0;
if(mode==CHANGE_DIR)
{
SetError(NO_FILE,propfind->ErrorText());
return MOVED;
}
if(propfind->ErrorFatal())
fileset_for_info->next();
Disconnect();
return MOVED;
}
if(propfind->Eof())
{
LogNote(9,"got EOF on PROPFIND reply");
const char *b;
int len;
propfind->Get(&b,&len);
Ref<FileSet> fs(HttpListInfo::ParseProps(b,len,GetCwd()));
propfind=0;
if(fs)
{
if(mode==CHANGE_DIR)
{
fs->rewind();
FileInfo *fi=fs->curr();
if(fi && fi->Has(fi->TYPE))
{
LogNote(9,"new-cwd: %s",fi->GetLongName());
new_cwd->is_file=(fi->filetype!=fi->DIRECTORY);
if(new_cwd->url.last_char()=='/' && new_cwd->is_file)
new_cwd->url.rtrim('/');
else if(new_cwd->url.last_char()!='/' && !new_cwd->is_file)
new_cwd->url.append('/');
}
}
else if(mode==ARRAY_INFO)
fileset_for_info->Merge(fs);
}
m=MOVED;
state=DONE;
if(mode==CHANGE_DIR)
{
cwd.Set(new_cwd);
cache->SetDirectory(this, "", !cwd.is_file);
return m;
}
else if(mode==ARRAY_INFO)
ProceedArrayInfo();
}
}
switch(state)
{
case DISCONNECTED:
if(mode==CLOSED || !hostname)
return m;
if(ModeIs(STORE) && pos>0 && entity_size>=0 && pos>=entity_size)
{
state=DONE;
return MOVED;
}
if(mode==ARRAY_INFO)
{
// check if we have anything to request
SendArrayInfoRequest();
if(!fileset_for_info->curr())
{
state=DONE;
return MOVED;
}
}
if(!hftp && mode==QUOTE_CMD && !special)
{
handle_quote_cmd:
if(file && !strncasecmp(file,"Set-Cookie ",11))
SetCookie(file+11);
else if(file && !strncasecmp(file,"POST ",5))
special=HTTP_POST;
else if(file && !strncasecmp(file,"COPY ",5))
special=HTTP_COPY;
else if(file && !strncasecmp(file,"MOVE ",5))
special=HTTP_MOVE;
else if(file && !strncasecmp(file,"PROPFIND ",9))
special=HTTP_PROPFIND;
else
{
SetError(NOT_SUPP,0);
return MOVED;
}
if(special)
{
// METHOD encoded_path data
const char *scan=file;
while(*scan && *scan!=' ')
scan++;
while(*scan==' ')
scan++;
file_url.set(https?"https://":"http://");
AppendHostEncoded(file_url,hostname);
if(portname)
{
file_url.append(':');
file_url.append_url_encoded(portname,URL_PORT_UNSAFE);
}
if(*scan!='/' && cwd)
{
if(cwd[0]!='/')
file_url.append('/');
file_url.append_url_encoded(cwd,URL_PATH_UNSAFE);
}
if(*scan!='/' && file_url.last_char()!='/')
file_url.append('/');
file_url.append(scan);
file_url.truncate_at(' ');
scan=strchr(scan,' ');
while(scan && *scan==' ')
scan++;
special_data.set(scan);
return MOVED;
}
state=DONE;
return MOVED;
}
if(!special && !ModeSupported())
{
SetError(NOT_SUPP);
return MOVED;
}
if(hftp)
{
if(!proxy)
{
// problem here: hftp cannot work without proxy
SetError(FATAL,_("ftp over http cannot work without proxy, set hftp:proxy."));
return MOVED;
}
}
// walk through Http classes and try to find identical idle session
// first try "easy" cases of session take-over.
for(int i=0; i<3; i++)
{
if(i>=2 && (connection_limit==0 || connection_limit>CountConnections()))
break;
GetBetterConnection(i);
if(state!=DISCONNECTED)
return MOVED;
}
if(!resolver && mode!=CONNECT_VERIFY && !ReconnectAllowed())
return m;
if(https)
m|=Resolve(HTTPS_DEFAULT_PORT,"https","tcp");
else
m|=Resolve(HTTP_DEFAULT_PORT,"http","tcp");
if(!peer)
return m;
if(mode==CONNECT_VERIFY)
return m;
if(!ReconnectAllowed())
return m;
if(!NextTry())
return MOVED;
retry_after=0;
res=SocketCreateTCP(peer[peer_curr].sa.sa_family);
if(res==-1)
{
saved_errno=errno;
if(peer_curr+1<peer.count())
{
peer_curr++;
retries--;
return MOVED;
}
if(NonFatalError(saved_errno))
return m;
SetError(SEE_ERRNO,xstring::format(
_("cannot create socket of address family %d"),
peer[peer_curr].sa.sa_family));
return MOVED;
}
conn=new Connection(res,hostname);
SayConnectingTo();
res=SocketConnect(conn->sock,&peer[peer_curr]);
if(res==-1 && errno!=EINPROGRESS)
{
saved_errno=errno;
NextPeer();
LogError(0,"connect: %s\n",strerror(saved_errno));
Disconnect();
if(NotSerious(saved_errno))
return MOVED;
goto system_error;
}
state=CONNECTING;
m=MOVED;
timeout_timer.Reset();
case CONNECTING:
res=Poll(conn->sock,POLLOUT,&error);
if(res==-1)
{
LogError(0,_("Socket error (%s) - reconnecting"),error);
Disconnect(error);
NextPeer();
return MOVED;
}
if(!(res&POLLOUT))
{
if(CheckTimeout())
{
NextPeer();
return MOVED;
}
Block(conn->sock,POLLOUT);
return m;
}
m=MOVED;
state=CONNECTED;
#if USE_SSL
if(proxy?!strncmp(proxy,"https://",8):https)
{
conn->MakeSSLBuffers();
}
else
#endif
{
conn->MakeBuffers();
#if USE_SSL
if(proxy && https)
{
// have to setup a tunnel.
xstring ehost;
AppendHostEncoded(ehost,hostname);
const char *port_to_use=portname?portname.get():HTTPS_DEFAULT_PORT;
const char *eport=url::encode(port_to_use,URL_PORT_UNSAFE);
Send("CONNECT %s:%s HTTP/1.1\r\n",ehost.get(),eport);
SendProxyAuth();
Send("\r\n");
tunnel_state=TUNNEL_WAITING;
state=RECEIVING_HEADER;
return MOVED;
}
#endif // USE_SSL
}
/*fallthrough*/
case CONNECTED:
if(mode==CONNECT_VERIFY)
return MOVED;
if(mode==QUOTE_CMD && !special)
goto handle_quote_cmd;
if(conn->recv_buf->Eof())
{
LogError(0,_("Peer closed connection"));
Disconnect();
return MOVED;
}
if(mode==CLOSED)
return m;
if(!special && !ModeSupported())
{
SetError(NOT_SUPP);
return MOVED;
}
ExpandTildeInCWD();
if(ModeIs(STORE) && pos>0 && entity_size>=0 && pos>=entity_size)
{
state=DONE;
return MOVED;
}
if(mode==ARRAY_INFO)
{
if(SendArrayInfoRequest()==0) {
// nothing to do
state=DONE;
return MOVED;
}
}
else
{
LogNote(9,_("Sending request..."));
SendRequest();
}
state=RECEIVING_HEADER;
m=MOVED;
if(ModeIs(STORE))
rate_limit=new RateLimit(hostname);
case RECEIVING_HEADER:
if(conn->send_buf->Error() || conn->recv_buf->Error())
{
if((ModeIs(STORE) || special) && status_code && !H_2XX(status_code))
goto pre_RECEIVING_BODY; // assume error.
handle_buf_error:
if(conn->send_buf->Error())
{
LogError(0,"send: %s",conn->send_buf->ErrorText());
if(conn->send_buf->ErrorFatal())
SetError(FATAL,conn->send_buf->ErrorText());
}
if(conn->recv_buf->Error())
{
LogError(0,"recv: %s",conn->recv_buf->ErrorText());
if(conn->recv_buf->ErrorFatal())
SetError(FATAL,conn->recv_buf->ErrorText());
}
Disconnect();
return MOVED;
}
timeout_timer.Reset(conn->send_buf->EventTime());
timeout_timer.Reset(conn->recv_buf->EventTime());
if(CheckTimeout())
return MOVED;
conn->recv_buf->Get(&buf,&len);
if(!buf)
{
// eof
LogError(0,_("Hit EOF while fetching headers"));
// workaround some broken servers
if(H_REDIRECTED(status_code) && location)
goto pre_RECEIVING_BODY;
Disconnect();
return MOVED;
}
if(len>0)
{
int eol_size;
const char *eol=find_eol(buf,len,&eol_size);
if(eol)
{
// empty line indicates end of headers.
if(eol==buf && status)
{
LogRecv(4,"");
conn->recv_buf->Skip(eol_size);
if(tunnel_state==TUNNEL_WAITING)
{
if(H_2XX(status_code))
{
#if USE_SSL
if(https)
conn->MakeSSLBuffers();
#endif
tunnel_state=TUNNEL_ESTABLISHED;
ResetRequestData();
state=CONNECTED;
return MOVED;
}
}
if(chunked_trailer)
{
chunked_trailer=false;
chunked=false;
if(propfind) {
// we avoid the DONE state since we have yet to handle propfind data
propfind->PutEOF();
state=CONNECTED;
} else
state=DONE;
return MOVED;
}
if(H_CONTINUE(status_code))
{
status.set(0);
status_code=0;
return MOVED;
}
if(mode==ARRAY_INFO)
{
if(!xstrcmp(last_method,"PROPFIND"))
{
if(H_UNSUPPORTED(status_code))
{
ResMgr::Set("http:use-propfind",hostname,"no");
use_propfind_now=false;
Disconnect();
DontSleep();
return MOVED;
}
goto pre_RECEIVING_BODY;
}
FileInfo *fi=fileset_for_info->curr();
if(H_REDIRECTED(status_code)) {
HandleRedirection();
if(location)
fi->SetRedirect(location);
} else if(H_2XX(status_code) && !fi->Has(fi->TYPE)) {
fi->SetType(last_uri.last_char()=='/'?fi->DIRECTORY:fi->NORMAL);
}
ProceedArrayInfo();
return MOVED;
}
else if(ModeIs(STORE) || ModeIs(MAKE_DIR) || sending_proppatch)
{
if((sent_eot || pos==entity_size || sending_proppatch) && H_2XX(status_code))
{
state=DONE;
Disconnect();
state=DONE;
if(ModeIs(STORE) && entity_date!=NO_DATE && !entity_date_set
&& use_propfind_now) {
// send PROPPATCH in a separate request.
sending_proppatch=true;
state=DISCONNECTED;
}
return MOVED;
}
if(H_2XX(status_code))
{
// should never happen
LogError(0,"Unexpected success, the server did not accept full request body");
Disconnect();
return MOVED;
}
// going to pre_RECEIVING_BODY to catch error
}
goto pre_RECEIVING_BODY;
}
len=eol-buf;
line.nset(buf,len);
conn->recv_buf->Skip(len+eol_size);
LogRecv(4,line);
m=MOVED;
if(status==0)
{
// it's status line
status.set(line);
int ver_major,ver_minor;
if(3!=sscanf(status,"HTTP/%d.%d %n%d",&ver_major,&ver_minor,
&status_consumed,&status_code))
{
// simple 0.9 ?
proto_version=0x09;
status_code=H_Ok;
LogError(0,_("Could not parse HTTP status line"));
if(ModeIs(STORE))
{
state=DONE;
Disconnect();
state=DONE;
return MOVED;
}
conn->recv_buf->UnSkip(len+eol_size);
goto pre_RECEIVING_BODY;
}
proto_version=(ver_major<<4)+ver_minor;
// HTTP/1.1 does keep-alive by default
if(proto_version>=0x11)
keep_alive=true;
if(!H_2XX(status_code))
{
if(H_CONTINUE(status_code))
return MOVED;
if(H_5XX(status_code)) // server failed, try another
NextPeer();
if(status_code==H_Gateway_Timeout)
{
const char *cc=Query("cache-control");
if(cc && strstr(cc,"only-if-cached"))
{
if(mode!=ARRAY_INFO)
{
SetError(NO_FILE,_("Object is not cached and http:cache-control has only-if-cached"));
return MOVED;
}
status_code=H_Not_Acceptable; // so that no retry will be attempted
}
}
// check for retriable codes
if(H_TRANSIENT(status_code))
{
Disconnect();
return MOVED;
}
if(status_code==H_Too_Many_Requests)
{
Disconnect();
if(retry_after)
reconnect_timer.StopDelayed(retry_after);
return MOVED;
}
if(mode==ARRAY_INFO)
TrySuccess();
return MOVED;
}
}
else
{
// header line.
char *colon=strchr(line.get_non_const(),':');
if(colon)
{
*colon=0; // terminate the header tag
const char *value=colon+1;
while(*value==' ')
value++;
HandleHeaderLine(line,value);
}
}
}
}
if(ModeIs(STORE) && (!status || H_CONTINUE(status_code)) && !sent_eot)
Block(conn->sock,POLLOUT);
return m;
pre_RECEIVING_BODY:
// 204 No Content
if(H_EMPTY(status_code) && body_size<0)
body_size=0;
if(H_REDIRECTED(status_code))
{
// check if it is redirection to the same server
// or to directory instead of file.
// FIXME.
}
if(H_REQUESTED_RANGE_NOT_SATISFIABLE(status_code))
{
// file is smaller than requested
state=DONE;
return MOVED;
}
if((status_code==H_Unauthorized && auth_scheme[HttpAuth::WWW])
|| (status_code==H_Proxy_Authentication_Required && auth_scheme[HttpAuth::PROXY])) {
// retry with authentication
retries--;
state=RECEIVING_BODY;
LogErrorText();
Disconnect();
DontSleep();
return MOVED;
}
if(!H_2XX(status_code))
{
xstring err;
int code=NO_FILE;
if(H_REDIRECTED(status_code))
{
HandleRedirection();
err.setf("%s (%s -> %s)",status+status_consumed,file.get(),
location?location.get():"nowhere");
code=FILE_MOVED;
}
else
{
const char *closure=file;
if(H_UNSUPPORTED(status_code) || status_code==H_Method_Not_Allowed)
{
if(H_UNSUPPORTED(status_code))
{
if(!xstrcmp(last_method,"PROPFIND"))
ResMgr::Set("http:use-propfind",hostname,"no");
if(!xstrcmp(last_method,"MKCOL"))
ResMgr::Set("http:use-mkcol",hostname,"no");
}
if(mode==CHANGE_DIR && !xstrcmp(last_method,"PROPFIND"))
{
use_propfind_now=false;
Disconnect();
DontSleep();
return MOVED;
}
code=NOT_SUPP;
closure=last_method;
}
if(closure && closure[0])
err.setf("%s (%s)",status+status_consumed,closure);
else
err.setf("%s (%s%s)",status+status_consumed,cwd.path.get(),
(last_char(cwd)=='/')?"":"/");
}
state=RECEIVING_BODY;
LogErrorText();
if(mode==ARRAY_INFO)
{
if(!H_TRANSIENT(status_code))
fileset_for_info->next();
Disconnect();
DontSleep();
}
else
SetError(code,err);
return MOVED;
}
if(!xstrcmp(last_method,"PROPFIND")
&& (mode==ARRAY_INFO || mode==CHANGE_DIR)) {
LogNote(9,"accepting XML for PROPFIND...");
propfind=new IOBufferFileAccess(this);
}
if(mode==CHANGE_DIR && !propfind)
{
cwd.Set(new_cwd);
cache->SetDirectory(this, "", !cwd.is_file);
state=DONE;
return MOVED;
}
// Many servers send application/x-gzip with x-gzip encoding,
// don't decode in such a case.
if(CompressedContentEncoding() && !CompressedContentType()
&& QueryBool("decode",hostname)) {
// inflated size is unknown beforehand
entity_size=NO_SIZE;
if(opt_size)
*opt_size=NO_SIZE;
// start the inflation
inflate=new DirectedBuffer(DirectedBuffer::GET);
inflate->SetTranslator(new DataInflator());
}
// sometimes it's possible to derive entity size from body size.
if(entity_size==NO_SIZE && body_size!=NO_SIZE
&& pos==0 && !ModeIs(STORE) && !ModeIs(MAKE_DIR) && !inflate) {
entity_size=body_size;
if(opt_size && H_2XX(status_code))
*opt_size=body_size;
}
LogNote(9,_("Receiving body..."));
rate_limit=new RateLimit(hostname);
if(real_pos<0) // assume Range: did not work
{
if(!ModeIs(STORE) && !ModeIs(MAKE_DIR) && body_size>=0)
{
entity_size=body_size;
if(opt_size && H_2XX(status_code))
*opt_size=entity_size;
}
real_pos=0;
}
state=RECEIVING_BODY;
m=MOVED;
/*passthrough*/
case RECEIVING_BODY:
if(conn->recv_buf->Error() || conn->send_buf->Error())
goto handle_buf_error;
if(conn->recv_buf->Size()>=rate_limit->BytesAllowedToGet())
{
conn->recv_buf->Suspend();
TimeoutS(1);
}
else if(conn->recv_buf->Size()>=max_buf)
{
conn->recv_buf->Suspend();
m=MOVED;
}
else
{
if(conn->recv_buf->IsSuspended())
{
conn->recv_buf->Resume();
if(conn->recv_buf->Size()>0 || (conn->recv_buf->Size()==0 && conn->recv_buf->Eof()))
m=MOVED;
}
timeout_timer.Reset(conn->send_buf->EventTime());
timeout_timer.Reset(conn->recv_buf->EventTime());
if(conn->recv_buf->Size()==0)
{
// check if ranges were emulated by squid
bool no_ranges_if_timeout=(bytes_received==0 && !seen_ranges_bytes);
if(CheckTimeout())
{
if(no_ranges_if_timeout)
{
no_ranges=true;
real_pos=0; // so that pget would know immediately.
}
return MOVED;
}
}
}
return m;
case DONE:
return m;
}
return m;
system_error:
assert(saved_errno!=0);
if(NonFatalError(saved_errno))
return m;
SetError(SEE_ERRNO,0);
Disconnect();
return MOVED;
}
void Http::HandleRedirection()
{
bool is_url=(location && url::is_url(location));
if(location && !is_url
&& mode==QUOTE_CMD && !strncasecmp(file,"POST ",5)
&& tunnel_state!=TUNNEL_WAITING)
{
const char *the_file=file;
const char *scan=file+5;
while(*scan==' ')
scan++;
char *the_post_file=alloca_strdup(scan);
char *space=strchr(the_post_file,' ');
if(space)
*space=0;
the_file=the_post_file;
char *new_location=alloca_strdup2(GetConnectURL(),
strlen(the_file)+strlen(location));
int p_ind=url::path_index(new_location);
if(location[0]=='/')
strcpy(new_location+p_ind,location);
else
{
if(the_file[0]=='/')
strcpy(new_location+p_ind,the_file);
else
{
char *slash=strrchr(new_location,'/');
strcpy(slash+1,the_file);
}
char *slash=strrchr(new_location,'/');
strcpy(slash+1,location);
}
location.set(new_location);
} else if(is_url && !hftp) {
ParsedURL url(location);
if(url.proto.eq(GetProto()) && !xstrcasecmp(url.host,hostname)
&& user && !url.user) {
// use the same user name after redirect to the same site.
url.user.set(user);
location.truncate();
url.CombineTo(location);
}
}
}
FileAccess *Http::New() { return new Http(); }
FileAccess *HFtp::New() { return new HFtp(); }
void Http::ClassInit()
{
// register the class
Register("http",Http::New);
Register("hftp",HFtp::New);
#if USE_SSL
Register("https",Https::New);
#endif
}
void Http::SuspendInternal()
{
super::SuspendInternal();
if(conn)
conn->SuspendInternal();
}
void Http::ResumeInternal()
{
if(conn)
conn->ResumeInternal();
super::ResumeInternal();
}
int Http::Read(Buffer *buf,int size)
{
if(Error())
return error_code;
if(mode==CLOSED)
return 0;
if(state==DONE)
return 0; // eof
int res=DO_AGAIN;
if(state==RECEIVING_BODY && real_pos>=0)
{
Enter(this);
res=_Read(buf,size);
if(res>0)
{
pos+=res;
if(rate_limit)
rate_limit->BytesGot(res);
TrySuccess();
}
Leave(this);
}
return res;
}
void Http::_Skip(int to_skip)
{
if(inflate)
inflate->Skip(to_skip);
else
conn->recv_buf->Skip(to_skip);
_UpdatePos(to_skip);
}
void Http::_UpdatePos(int to_skip)
{
if(!inflate) {
if(chunked)
chunk_pos+=to_skip;
bytes_received+=to_skip;
}
real_pos+=to_skip;
}
int Http::_Read(Buffer *buf,int size)
{
const char *buf1;
int size1;
Buffer *src_buf=conn->recv_buf.get_non_const();
get_again:
if(conn->recv_buf->Size()==0 && conn->recv_buf->Error())
{
LogError(0,"recv: %s",conn->recv_buf->ErrorText());
if(conn->recv_buf->ErrorFatal())
SetError(FATAL,conn->recv_buf->ErrorText());
Disconnect();
return DO_AGAIN;
}
conn->recv_buf->Get(&buf1,&size1);
if(buf1==0) // eof
{
LogNote(9,_("Hit EOF"));
if(bytes_received<body_size || chunked)
{
LogError(0,_("Received not enough data, retrying"));
Disconnect();
return DO_AGAIN;
}
return 0;
}
if(!chunked)
{
if(body_size>=0 && bytes_received>=body_size
&& (!inflate || inflate->Size()==0))
{
LogNote(9,_("Received all"));
return 0; // all received
}
if(entity_size>=0 && pos>=entity_size)
{
LogNote(9,_("Received all (total)"));
return 0;
}
}
if(size1==0 && (!inflate || inflate->Size()==0))
return DO_AGAIN;
if(chunked && size1>0)
{
if(chunked_trailer && state==RECEIVING_HEADER)
return DO_AGAIN;
const char *nl;
if(chunk_size==CHUNK_SIZE_UNKNOWN) // expecting first/next chunk
{
nl=(const char*)memchr(buf1,'\n',size1);
if(nl==0) // not yet
{
not_yet:
if(conn->recv_buf->Eof())
Disconnect(); // connection closed too early
return DO_AGAIN;
}
if(!is_ascii_xdigit(*buf1)
|| sscanf(buf1,"%lx",&chunk_size)!=1)
{
Fatal(_("chunked format violated"));
return FATAL;
}
conn->recv_buf->Skip(nl-buf1+1);
chunk_pos=0;
LogNote(9,"next chunk size: %ld",chunk_size);
goto get_again;
}
if(chunk_size==0) // eof
{
LogNote(9,_("Received last chunk"));
// headers may follow
chunked_trailer=true;
state=RECEIVING_HEADER;
body_size=bytes_received;
Timeout(0);
return DO_AGAIN;
}
if(chunk_pos==chunk_size)
{
if(size1<2)
goto not_yet;
if(buf1[0]!='\r' || buf1[1]!='\n')
{
Fatal(_("chunked format violated"));
return FATAL;
}
conn->recv_buf->Skip(2);
chunk_size=CHUNK_SIZE_UNKNOWN;
goto get_again;
}
// ok, now we may get portion of data
if(size1>chunk_size-chunk_pos)
size1=chunk_size-chunk_pos;
}
if(!chunked)
{
// limit by body_size.
if(body_size>=0 && size1+bytes_received>=body_size)
size1=body_size-bytes_received;
}
int bytes_allowed=0x10000000;
if(rate_limit)
bytes_allowed=rate_limit->BytesAllowedToGet();
if(inflate) {
// do the inflation, if there are not enough inflated data
if(size1>bytes_allowed)
size1=bytes_allowed;
if(inflate->Size()<size && size1>0) {
inflate->PutTranslated(buf1,size1);
conn->recv_buf->Skip(size1);
if(chunked)
chunk_pos+=size1;
bytes_received+=size1;
if(inflate->Error())
SetError(FATAL,inflate->ErrorText());
}
inflate->Get(&buf1,&size1);
src_buf=inflate.get_non_const();
} else {
if(size1>bytes_allowed)
size1=bytes_allowed;
}
if(size1==0)
return DO_AGAIN;
if(norest_manual && real_pos==0 && pos>0)
return DO_AGAIN;
if(real_pos<pos)
{
off_t to_skip=pos-real_pos;
if(to_skip>size1)
to_skip=size1;
_Skip(to_skip);
goto get_again;
}
if(size>size1)
size=size1;
size=buf->MoveDataHere(src_buf,size);
_UpdatePos(size);
return size;
}
int Http::Done()
{
if(mode==CLOSED)
return OK;
if(Error())
return error_code;
if(state==DONE)
return OK;
if(mode==CONNECT_VERIFY && (peer || conn))
return OK;
if((mode==REMOVE || mode==REMOVE_DIR || mode==RENAME)
&& state==RECEIVING_BODY)
return OK;
return IN_PROGRESS;
}
int Http::Buffered()
{
if(!ModeIs(STORE) || !conn || !conn->send_buf)
return 0;
return conn->send_buf->Size()+SocketBuffered(conn->sock);
}
int Http::Write(const void *buf,int size)
{
if(!ModeIs(STORE))
return(0);
Resume();
Do();
if(Error())
return(error_code);
if(state!=RECEIVING_HEADER || status!=0 || conn->send_buf->Size()!=0)
return DO_AGAIN;
{
int allowed=rate_limit->BytesAllowedToPut();
if(allowed==0)
return DO_AGAIN;
if(size>allowed)
size=allowed;
}
if(size+conn->send_buf->Size()>=max_buf)
size=max_buf-conn->send_buf->Size();
if(entity_size!=NO_SIZE && pos+size>entity_size)
{
size=entity_size-pos;
// tried to write more than originally requested. Make it retry with Open:
if(size==0)
return STORE_FAILED;
}
if(size<=0)
return 0;
conn->send_buf->Put((const char*)buf,size);
if(retries>0 && conn->send_buf->GetPos()-conn->send_buf->Size()>Buffered()+0x1000)
TrySuccess();
rate_limit->BytesPut(size);
pos+=size;
real_pos+=size;
return(size);
}
int Http::SendEOT()
{
if(sent_eot)
return OK;
if(Error())
return(error_code);
if(ModeIs(STORE))
{
if(state==RECEIVING_HEADER && conn->send_buf->Size()==0)
{
if(entity_size==NO_SIZE || pos<entity_size)
{
shutdown(conn->sock,1);
keep_alive=false;
}
sent_eot=true;
return(OK);
}
return(DO_AGAIN);
}
return(OK);
}
int Http::StoreStatus()
{
if(!sent_eot && state==RECEIVING_HEADER)
SendEOT();
return Done();
}
const char *Http::CurrentStatus()
{
switch(state)
{
case DISCONNECTED:
if(hostname)
{
if(resolver)
return(_("Resolving host address..."));
if(!ReconnectAllowed())
return DelayingMessage();
}
return "";
case CONNECTING:
return(_("Connecting..."));
case CONNECTED:
return(_("Connection idle"));
case RECEIVING_HEADER:
if(ModeIs(STORE) && !sent_eot && !status)
return(_("Sending data"));
if(tunnel_state==TUNNEL_WAITING)
return(_("Connecting..."));
if(!status)
return(_("Waiting for response..."));
return(_("Fetching headers..."));
case RECEIVING_BODY:
return(_("Receiving data"));
case DONE:
return "";
}
abort();
}
void Http::Reconfig(const char *name)
{
const char *c=hostname;
super::Reconfig(name);
no_cache = !QueryBool("cache",c);
if(!hftp && NoProxy(hostname))
SetProxy(0);
else
{
const char *p=0;
if(hftp && vproto && !strcmp(vproto,"ftp"))
{
p=ResMgr::Query("ftp:proxy",c);
if(p && strncmp(p,"http://",7) && strncmp(p,"https://",8))
p=0;
}
if(!p)
{
if(https)
p=ResMgr::Query("https:proxy",c);
else
p=Query("proxy",c);
// if no hftp:proxy is specified, try http:proxy.
if(hftp && !p)
p=ResMgr::Query("http:proxy",c);
}
SetProxy(p);
}
if(conn)
SetSocketBuffer(conn->sock);
if(proxy && proxy_port==0)
proxy_port.set(HTTP_DEFAULT_PROXY_PORT);
user_agent=ResMgr::Query("http:user-agent",c);
use_propfind_now=(use_propfind_now && QueryBool("use-propfind",c));
no_ranges=(no_ranges || !QueryBool("use-range",hostname));
if(QueryBool("use-allprop",c)) {
allprop.set( // PROPFIND request
"<?xml version=\"1.0\" ?>"
"<propfind xmlns=\"DAV:\">"
"<allprop/>"
"</propfind>\r\n");
} else {
allprop.unset();
}
if(!user || !pass) {
// get auth info from http:authorization setting
const char *auth_c=Query("authorization",hostname);
if(auth_c && *auth_c) {
char *auth=alloca_strdup(auth_c);
char *colon=strchr(auth,':');
if(colon) {
*colon=0;
auth_user.set(auth);
auth_pass.set(colon+1);
}
}
}
}
bool Http::SameSiteAs(const FileAccess *fa) const
{
if(!SameProtoAs(fa))
return false;
Http *o=(Http*)fa;
return(!xstrcasecmp(hostname,o->hostname) && !xstrcmp(portname,o->portname)
&& !xstrcmp(user,o->user) && !xstrcmp(pass,o->pass));
}
bool Http::SameLocationAs(const FileAccess *fa) const
{
if(!SameSiteAs(fa))
return false;
Http *o=(Http*)fa;
if(cwd!=o->cwd)
return false;
return true;
}
void Http::ResetLocationData()
{
super::ResetLocationData();
Reconfig();
state=DISCONNECTED;
use_propfind_now=QueryBool("use-propfind",hostname);
no_ranges=!QueryBool("use-range",hostname);
}
DirList *Http::MakeDirList(ArgV *args)
{
return new HttpDirList(this,args);
}
#include "FileGlob.h"
Glob *Http::MakeGlob(const char *pattern)
{
return new GenericGlob(this,pattern);
}
ListInfo *Http::MakeListInfo(const char *path)
{
return new HttpListInfo(this,path);
}
bool Http::CookieClosureMatch(const char *closure_c,
const char *hostname,const char *efile)
{
if(!closure_c)
return true;
char *closure=alloca_strdup2(closure_c,1);
char *path=0;
char *scan=closure;
for(;;)
{
char *slash=strchr(scan,';');
if(!slash)
break;
*slash++=0;
while(*slash && *slash==' ')
slash++;
if(!strncmp(slash,"path=",5))
path=slash+5;
else if(!strncmp(slash,"secure",6) && (slash[6]==';' || slash[6]==0))
{
if(!https)
return false;
}
}
if(closure[0] && 0!=fnmatch(closure,hostname,FNM_PATHNAME))
return false;
if(!path)
return true;
int path_len=strlen(path);
if(path_len>0 && path[path_len-1]=='/')
path_len--;
if(!strncmp(efile,path,path_len)
&& (efile[path_len]==0 || efile[path_len]=='/'))
return true;
return false;
}
void Http::CookieMerge(xstring &all,const char *cookie_c)
{
char *value=alloca_strdup(cookie_c);
for(char *entry=strtok(value,";"); entry; entry=strtok(0,";"))
{
if(*entry==' ')
entry++;
if(*entry==0)
break;
if(!strncasecmp(entry,"path=",5)
|| !strncasecmp(entry,"expires=",8)
|| !strncasecmp(entry,"domain=",7)
|| (!strncasecmp(entry,"secure",6)
&& (entry[6]==' ' || entry[6]==0 || entry[6]==';')))
continue; // filter out path= expires= domain= secure
char *c_name=entry;
char *c_value=strchr(entry,'=');
if(c_value)
*c_value++=0;
else
c_value=c_name, c_name=0;
int c_name_len=xstrlen(c_name);
for(unsigned i=all.skip_all(0,' '); i<all.length(); i=all.skip_all(i+1,' '))
{
const char *scan=all+i;
const char *semicolon=strchr(scan,';');
const char *eq=strchr(scan,'=');
if(semicolon && eq>semicolon)
eq=0;
if((eq==0 && c_name==0)
|| (eq-scan==c_name_len && !strncmp(scan,c_name,c_name_len)))
{
// remove old cookie.
if(!semicolon)
all.truncate(i);
else
all.set_substr(i,all.skip_all(semicolon+1-all,' ')-i,"",0);
break;
}
if(!semicolon)
break;
i=semicolon+1-all;
}
// append cookie.
all.rtrim(' ');
all.rtrim(';');
int c_len=all.length();
if(c_len>0 && all[c_len-1]!=';')
all.append("; ");
if(c_name)
all.vappend(c_name,"=",c_value,NULL);
else
all.append(c_value);
}
}
void Http::MakeCookie(xstring &all_cookies,const char *hostname,const char *efile)
{
Resource *scan=0;
const char *closure;
for(;;)
{
const char *cookie=ResMgr::QueryNext("http:cookie",&closure,&scan);
if(cookie==0)
break;
if(!CookieClosureMatch(closure,hostname,efile))
continue;
CookieMerge(all_cookies,cookie);
}
}
void Http::SetCookie(const char *value_const)
{
char *value=alloca_strdup(value_const);
const char *domain=hostname;
const char *path=0;
bool secure=false;
for(char *entry=strtok(value,";"); entry; entry=strtok(0,";"))
{
while(*entry==' ') // skip spaces.
entry++;
if(*entry==0)
break;
if(!strncasecmp(entry,"expires=",8))
continue; // not used yet (FIXME)
if(!strncasecmp(entry,"secure",6)
&& (entry[6]==' ' || entry[6]==0 || entry[6]==';'))
{
secure=true;
continue;
}
if(!strncasecmp(entry,"path=",5))
{
path=alloca_strdup(entry+5);
continue;
}
if(!strncasecmp(entry,"domain=",7))
{
char *new_domain=alloca_strdup(entry+6);
if(new_domain[1]=='.')
new_domain[0]='*';
else
new_domain++;
char *end=strchr(new_domain,';');
if(end)
*end=0;
domain=new_domain;
continue;
}
}
xstring closure(domain);
if(path && path[0] && strcmp(path,"/"))
closure.append(";path=").append(path);
if(secure)
closure.append(";secure");
xstring c(Query("cookie",closure));
CookieMerge(c,value_const);
ResMgr::Set("http:cookie",closure,c);
}
#if USE_SSL
#undef super
#define super Http
Https::Https()
{
https=true;
res_prefix="http";
}
Https::~Https()
{
}
Https::Https(const Https *o) : super(o)
{
https=true;
res_prefix="http";
Reconfig(0);
}
FileAccess *Https::New(){ return new Https();}
void Http::Connection::MakeSSLBuffers()
{
ssl=new lftp_ssl(sock,lftp_ssl::CLIENT,closure);
ssl->load_keys();
IOBufferSSL *send_buf_ssl=new IOBufferSSL(ssl,IOBuffer::PUT);
IOBufferSSL *recv_buf_ssl=new IOBufferSSL(ssl,IOBuffer::GET);
send_buf=send_buf_ssl;
recv_buf=recv_buf_ssl;
}
#endif
#undef super
#define super Http
HFtp::HFtp()
{
hftp=true;
default_cwd="~";
Reconfig(0);
}
HFtp::~HFtp()
{
}
HFtp::HFtp(const HFtp *o) : super(o)
{
hftp=true;
Reconfig(0);
}
void HFtp::Login(const char *u,const char *p)
{
super::Login(u,p);
if(u)
{
home.Set("~");
cwd.Set(home,false,0);
}
}
void HFtp::Reconfig(const char *name)
{
super::Reconfig(name);
use_head=QueryBool("use-head");
}
void Http::LogErrorText()
{
if(!conn || !conn->recv_buf)
return;
conn->recv_buf->Roll();
int size=conn->recv_buf->Size();
if(size==0)
return;
Buffer tmpbuf;
size=_Read(&tmpbuf,size);
if(size<=0)
return;
tmpbuf.SpaceAdd(size);
const char *buf0=tmpbuf.Get();
char *buf=alloca_strdup(buf0);
remove_tags(buf);
for(char *line=strtok(buf,"\n"); line; line=strtok(0,"\n")) {
rtrim(line);
if(*line)
Log::global->Format(4,"<--* %s\n",line);
}
}
/* The functions http_atotm and check_end are taken from wget */
#define ISSPACE(c) is_ascii_space((c))
#define ISDIGIT(c) is_ascii_digit((c))
/* Check whether the result of strptime() indicates success.
strptime() returns the pointer to how far it got to in the string.
The processing has been successful if the string is at `GMT' or
`+X', or at the end of the string.
In extended regexp parlance, the function returns 1 if P matches
"^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (a valid result of
strptime()) is considered a failure and 0 is returned. */
static int
check_end (const char *p)
{
if (!p)
return 0;
while (ISSPACE (*p))
++p;
if (!*p
|| (p[0] == 'G' && p[1] == 'M' && p[2] == 'T')
|| (p[0] == 'U' && p[1] == 'T' && p[2] == 'C')
|| ((p[0] == '+' || p[1] == '-') && ISDIGIT (p[1])))
return 1;
else
return 0;
}
/* Convert TIME_STRING time to time_t. TIME_STRING can be in any of
the three formats RFC2068 allows the HTTP servers to emit --
RFC1123-date, RFC850-date or asctime-date. Timezones are ignored,
and should be GMT.
We use strptime() to recognize various dates, which makes it a
little bit slacker than the RFC1123/RFC850/asctime (e.g. it always
allows shortened dates and months, one-digit days, etc.). It also
allows more than one space anywhere where the specs require one SP.
The routine should probably be even more forgiving (as recommended
by RFC2068), but I do not have the time to write one.
Return the computed time_t representation, or ATOTM_ERROR if all the
schemes fail.
Needless to say, what we *really* need here is something like
Marcus Hennecke's atotm(), which is forgiving, fast, to-the-point,
and does not use strptime(). atotm() is to be found in the sources
of `phttpd', a little-known HTTP server written by Peter Erikson. */
time_t
Http::atotm (const char *time_string)
{
struct tm t;
/* Roger Beeman says: "This function dynamically allocates struct tm
t, but does no initialization. The only field that actually
needs initialization is tm_isdst, since the others will be set by
strptime. Since strptime does not set tm_isdst, it will return
the data structure with whatever data was in tm_isdst to begin
with. For those of us in timezones where DST can occur, there
can be a one hour shift depending on the previous contents of the
data area where the data structure is allocated." */
t.tm_isdst = -1;
/* Note that under foreign locales Solaris strptime() fails to
recognize English dates, which renders this function useless. I
assume that other non-GNU strptime's are plagued by the same
disease. We solve this by setting only LC_MESSAGES in
i18n_initialize(), instead of LC_ALL.
Another solution could be to temporarily set locale to C, invoke
strptime(), and restore it back. This is slow and dirty,
however, and locale support other than LC_MESSAGES can mess other
things, so I rather chose to stick with just setting LC_MESSAGES.
Also note that none of this is necessary under GNU strptime(),
because it recognizes both international and local dates. */
/* NOTE: We don't use `%n' for white space, as OSF's strptime uses
it to eat all white space up to (and including) a newline, and
the function fails if there is no newline (!).
Let's hope all strptime() implementations use ` ' to skip *all*
whitespace instead of just one (it works that way on all the
systems I've tested it on). */
time_t ut=ATOTM_ERROR;
setlocale(LC_TIME,"C"); // we need english month and week day names
/* RFC1123: Thu, 29 Jan 1998 22:12:57 */
if (check_end (strptime (time_string, "%a, %d %b %Y %T", &t)))
ut=mktime_from_utc (&t);
/* RFC850: Thu, 29-Jan-98 22:12:57 */
else if (check_end (strptime (time_string, "%a, %d-%b-%y %T", &t)))
ut=mktime_from_utc (&t);
/* asctime: Thu Jan 29 22:12:57 1998 */
else if (check_end (strptime (time_string, "%a %b %d %T %Y", &t)))
ut=mktime_from_utc (&t);
setlocale(LC_TIME,""); // restore locale
return ut;
}
bool Http::IsCompressed(const char *s)
{
static const char *const values[] = {
"x-gzip", "gzip", "deflate", "compress", "x-compress", NULL
};
for(const char *const *v=values; *v; v++)
if(!strcmp(s,*v))
return true;
return false;
}
bool Http::CompressedContentEncoding() const
{
return content_encoding && IsCompressed(content_encoding);
}
bool Http::CompressedContentType() const
{
if(file.ends_with(".gz") || file.ends_with(".Z") || file.ends_with(".tgz"))
return true;
static const char app[]="application/";
return entity_content_type && entity_content_type.begins_with(app)
&& IsCompressed(entity_content_type+sizeof(app)-1);
}
#include "modconfig.h"
#ifdef MODULE_PROTO_HTTP
void module_init()
{
Http::ClassInit();
}
#endif