/*
* $LynxId: HTTP.c,v 1.175 2018/05/04 20:07:43 Elliot.Thomas Exp $
*
* HyperText Tranfer Protocol - Client implementation HTTP.c
* ==========================
* Modified:
* 27 Jan 1994 PDM Added Ari Luotonen's Fix for Reload when using proxy
* servers.
* 28 Apr 1997 AJL,FM Do Proxy Authorisation.
*/
#include <HTUtils.h>
#include <HTTP.h>
#include <LYUtils.h>
#ifdef USE_SSL
#include <HTNews.h>
#endif
#define HTTP_PORT 80
#define HTTPS_PORT 443
#define SNEWS_PORT 563
#define INIT_LINE_SIZE 1536 /* Start with line buffer this big */
#define LINE_EXTEND_THRESH 256 /* Minimum read size */
#define VERSION_LENGTH 20 /* for returned protocol version */
#include <HTParse.h>
#include <HTTCP.h>
#include <HTFormat.h>
#include <HTFile.h>
#include <HTAlert.h>
#include <HTMIME.h>
#include <HTML.h>
#include <HTInit.h>
#include <HTAABrow.h>
#include <HTAccess.h> /* Are we using an HTTP gateway? */
#include <LYCookie.h>
#include <LYGlobalDefs.h>
#include <GridText.h>
#include <LYStrings.h>
#include <LYUtils.h>
#include <LYrcFile.h>
#include <LYLeaks.h>
#include <LYCurses.h>
#ifdef USE_SSL
#ifdef USE_OPENSSL_INCL
#include <openssl/x509v3.h>
#endif
#if defined(LIBRESSL_VERSION_NUMBER)
/* OpenSSL and LibreSSL version numbers do not correspond */
#if LIBRESSL_VERSION_NUMBER >= 0x2060100fL
#define SSL_set_no_TLSV1() SSL_set_min_proto_version(handle, TLS1_1_VERSION)
#endif
#elif defined(OPENSSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L)
#define SSLEAY_VERSION_NUMBER OPENSSL_VERSION_NUMBER
#undef SSL_load_error_strings
#undef SSLeay_add_ssl_algorithms
#define ASN1_STRING_data ASN1_STRING_get0_data
#define TLS_client_method() SSLv23_client_method()
#define SSL_load_error_strings() /* nothing */
#define SSLeay_add_ssl_algorithms() /* nothing */
#define SSL_set_no_TLSV1() SSL_set_min_proto_version(handle, TLS1_1_VERSION)
#elif defined(SSLEAY_VERSION_NUMBER)
#define TLS_client_method() SSLv23_client_method()
#endif
#ifndef SSL_set_no_TLSV1
#define SSL_set_no_TLSV1() SSL_set_options(handle, SSL_OP_NO_TLSv1)
#endif
#ifdef USE_GNUTLS_INCL
#include <gnutls/x509.h>
#endif
#endif /* USE_SSL */
BOOLEAN reloading = FALSE; /* Reloading => send no-cache pragma to proxy */
char *redirecting_url = NULL; /* Location: value. */
BOOL permanent_redirection = FALSE; /* Got 301 status? */
BOOL redirect_post_content = FALSE; /* Don't convert to GET? */
#ifdef USE_SSL
SSL_CTX *ssl_ctx = NULL; /* SSL ctx */
SSL *SSL_handle = NULL;
static int ssl_okay;
static void free_ssl_ctx(void)
{
if (ssl_ctx != NULL)
SSL_CTX_free(ssl_ctx);
}
static BOOL needs_limit(const char *actual)
{
return ((int) strlen(actual) > LYcols - 7) ? TRUE : FALSE;
}
static char *limited_string(const char *source, const char *actual)
{
int limit = ((int) strlen(source)
- ((int) strlen(actual) - (LYcols - 10)));
char *temp = NULL;
StrAllocCopy(temp, source);
if (limit < 0)
limit = 0;
strcpy(temp + limit, "...");
return temp;
}
/*
* If the error message is too long to fit in the line, truncate that to fit
* within the limits for prompting.
*/
static void SSL_single_prompt(char **target, const char *source)
{
HTSprintf0(target, SSL_FORCED_PROMPT, source);
if (needs_limit(*target)) {
char *temp = limited_string(source, *target);
*target = NULL;
HTSprintf0(target, SSL_FORCED_PROMPT, temp);
free(temp);
}
}
static void SSL_double_prompt(char **target, const char *format, const char
*arg1, const char *arg2)
{
HTSprintf0(target, format, arg1, arg2);
if (needs_limit(*target)) {
char *parg2 = limited_string(arg2, *target);
*target = NULL;
HTSprintf0(target, format, arg1, parg2);
if (needs_limit(*target)) {
char *parg1 = limited_string(arg1, *target);
*target = NULL;
HTSprintf0(target, format, parg1, parg2);
free(parg1);
}
free(parg2);
}
}
static int HTSSLCallback(int preverify_ok, X509_STORE_CTX * x509_ctx GCC_UNUSED)
{
char *msg = NULL;
int result = 1;
#ifdef USE_X509_SUPPORT
HTSprintf0(&msg,
gettext("SSL callback:%s, preverify_ok=%d, ssl_okay=%d"),
X509_verify_cert_error_string((long) X509_STORE_CTX_get_error(x509_ctx)),
preverify_ok, ssl_okay);
_HTProgress(msg);
FREE(msg);
#endif
#ifndef USE_NSS_COMPAT_INCL
if (!(preverify_ok || ssl_okay || ssl_noprompt)) {
#ifdef USE_X509_SUPPORT
SSL_single_prompt(&msg,
X509_verify_cert_error_string((long)
X509_STORE_CTX_get_error(x509_ctx)));
if (HTForcedPrompt(ssl_noprompt, msg, NO))
ssl_okay = 1;
else
result = 0;
#endif
FREE(msg);
}
#endif
return result;
}
SSL *HTGetSSLHandle(void)
{
#ifdef USE_GNUTLS_INCL
static char *certfile = NULL;
#endif
static char *client_keyfile = NULL;
static char *client_certfile = NULL;
if (ssl_ctx == NULL) {
/*
* First time only.
*/
#if SSLEAY_VERSION_NUMBER < 0x0800
if ((ssl_ctx = SSL_CTX_new()) != NULL) {
X509_set_default_verify_paths(ssl_ctx->cert);
}
#else
SSLeay_add_ssl_algorithms();
if ((ssl_ctx = SSL_CTX_new(TLS_client_method())) != NULL) {
#ifdef SSL_OP_NO_SSLv2
SSL_CTX_set_options(ssl_ctx, SSL_OP_ALL | SSL_OP_NO_SSLv2);
#else
SSL_CTX_set_options(ssl_ctx, SSL_OP_ALL);
#endif
#ifdef SSL_OP_NO_COMPRESSION
SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_COMPRESSION);
#endif
#ifdef SSL_MODE_AUTO_RETRY
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
#endif
#ifdef SSL_MODE_RELEASE_BUFFERS
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
#endif
SSL_CTX_set_default_verify_paths(ssl_ctx);
SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, HTSSLCallback);
}
#endif /* SSLEAY_VERSION_NUMBER < 0x0800 */
#if defined(USE_PROGRAM_DIR) & !defined(USE_GNUTLS_INCL)
if (ssl_ctx != NULL) {
X509_LOOKUP *lookup;
lookup = X509_STORE_add_lookup(ssl_ctx->cert_store,
X509_LOOKUP_file());
if (lookup != NULL) {
char *certfile = NULL;
HTSprintf0(&certfile, "%s\\cert.pem", program_dir);
X509_LOOKUP_load_file(lookup, certfile, X509_FILETYPE_PEM);
FREE(certfile);
}
}
#endif
#ifdef USE_GNUTLS_INCL
if ((certfile = LYGetEnv("SSL_CERT_FILE")) != NULL) {
CTRACE((tfp,
"HTGetSSLHandle: certfile is set to %s by SSL_CERT_FILE\n",
certfile));
} else {
if (non_empty(SSL_cert_file)) {
certfile = SSL_cert_file;
CTRACE((tfp,
"HTGetSSLHandle: certfile is set to %s by config SSL_CERT_FILE\n",
certfile));
}
#if defined(USE_PROGRAM_DIR)
else {
HTSprintf0(&(certfile), "%s\\cert.pem", program_dir);
CTRACE((tfp,
"HTGetSSLHandle: certfile is set to %s by installed directory\n", certfile));
}
#endif
}
#endif
atexit(free_ssl_ctx);
}
if (non_empty(SSL_client_key_file)) {
client_keyfile = SSL_client_key_file;
CTRACE((tfp,
"HTGetSSLHandle: client key file is set to %s by config SSL_CLIENT_KEY_FILE\n",
client_keyfile));
}
if (non_empty(SSL_client_cert_file)) {
client_certfile = SSL_client_cert_file;
CTRACE((tfp,
"HTGetSSLHandle: client cert file is set to %s by config SSL_CLIENT_CERT_FILE\n",
client_certfile));
}
#ifdef USE_GNUTLS_INCL
ssl_ctx->certfile = certfile;
ssl_ctx->certfile_type = GNUTLS_X509_FMT_PEM;
ssl_ctx->client_keyfile = client_keyfile;
ssl_ctx->client_keyfile_type = GNUTLS_X509_FMT_PEM;
ssl_ctx->client_certfile = client_certfile;
ssl_ctx->client_certfile_type = GNUTLS_X509_FMT_PEM;
#elif SSLEAY_VERSION_NUMBER >= 0x0930
if (client_certfile != NULL) {
if (client_keyfile == NULL) {
client_keyfile = client_certfile;
}
SSL_CTX_use_certificate_chain_file(ssl_ctx, client_certfile);
SSL_CTX_use_PrivateKey_file(ssl_ctx, client_keyfile, SSL_FILETYPE_PEM);
}
#endif
ssl_okay = 0;
return (SSL_new(ssl_ctx));
}
void HTSSLInitPRNG(void)
{
#if SSLEAY_VERSION_NUMBER >= 0x00905100
if (RAND_status() == 0) {
char rand_file[256];
time_t t;
long l, seed;
#ifndef _WINDOWS
pid_t pid;
#else
DWORD pid;
#endif
t = time(NULL);
#ifndef _WINDOWS
pid = getpid();
#else
pid = GetCurrentThreadId();
#endif
RAND_file_name(rand_file, 256L);
CTRACE((tfp, "HTTP: Seeding PRNG\n"));
/* Seed as much as 1024 bytes from RAND_file_name */
RAND_load_file(rand_file, 1024L);
/* Seed in time (mod_ssl does this) */
RAND_seed((unsigned char *) &t, (int) sizeof(time_t));
/* Seed in pid (mod_ssl does this) */
RAND_seed((unsigned char *) &pid, (int) sizeof(pid));
/* Initialize system's random number generator */
RAND_bytes((unsigned char *) &seed, (int) sizeof(long));
lynx_srand((unsigned) seed);
while (RAND_status() == 0) {
/* Repeatedly seed the PRNG using the system's random number generator until it has been seeded with enough data */
l = lynx_rand();
RAND_seed((unsigned char *) &l, (int) sizeof(long));
}
/* Write a rand_file */
RAND_write_file(rand_file);
}
#endif /* SSLEAY_VERSION_NUMBER >= 0x00905100 */
return;
}
#define HTTP_NETREAD(sock, buff, size, handle) \
(handle \
? SSL_read(handle, buff, size) \
: NETREAD(sock, buff, size))
#define HTTP_NETWRITE(sock, buff, size, handle) \
(handle \
? SSL_write(handle, buff, size) \
: NETWRITE(sock, buff, size))
#define HTTP_NETCLOSE(sock, handle) \
{ (void)NETCLOSE(sock); \
if (handle) \
SSL_free(handle); \
SSL_handle = handle = NULL; \
}
#else
#define HTTP_NETREAD(a, b, c, d) NETREAD(a, b, c)
#define HTTP_NETWRITE(a, b, c, d) NETWRITE(a, b, c)
#define HTTP_NETCLOSE(a, b) (void)NETCLOSE(a)
#endif /* USE_SSL */
#ifdef _WINDOWS /* 1997/11/06 (Thu) 13:00:08 */
#define BOX_TITLE "Lynx " __FILE__
#define BOX_FLAG (MB_ICONINFORMATION | MB_SETFOREGROUND)
typedef struct {
int fd;
char *buf;
int len;
} recv_data_t;
int ws_read_per_sec = 0;
static int ws_errno = 0;
static DWORD g_total_times = 0;
static DWORD g_total_bytes = 0;
/* The same like read, but takes care of EINTR and uses select to
timeout the stale connections. */
static int ws_read(int fd, char *buf, int len)
{
int res;
int retry = 3;
do {
res = recv(fd, buf, len, 0);
if (WSAEWOULDBLOCK == WSAGetLastError()) {
Sleep(100);
if (retry-- > 0)
continue;
}
} while (res == SOCKET_ERROR && SOCKET_ERRNO == EINTR);
return res;
}
#define DWORD_ERR ((DWORD)-1)
static DWORD __stdcall _thread_func(void *p)
{
DWORD result;
int i, val;
recv_data_t *q = (recv_data_t *) p;
i = 0;
i++;
val = ws_read(q->fd, q->buf, q->len);
if (val == SOCKET_ERROR) {
ws_errno = WSAGetLastError();
#if 0
char buff[256];
sprintf(buff, "Thread read: %d, error (%ld), fd = %d, len = %d",
i, ws_errno, q->fd, q->len);
MessageBox(NULL, buff, BOX_TITLE, BOX_FLAG);
#endif
result = DWORD_ERR;
} else {
result = val;
}
return result;
}
/* The same like read, but takes care of EINTR and uses select to
timeout the stale connections. */
int ws_netread(int fd, char *buf, int len)
{
int i;
char buff[256];
/* 1998/03/30 (Mon) 09:01:21 */
HANDLE hThread;
DWORD dwThreadID;
DWORD exitcode = 0;
DWORD ret_val = DWORD_ERR;
DWORD val, process_time, now_TickCount, save_TickCount;
static recv_data_t para;
#define TICK 5
#define STACK_SIZE 0x2000uL
EnterCriticalSection(&critSec_READ);
para.fd = fd;
para.buf = buf;
para.len = len;
ws_read_per_sec = 0;
save_TickCount = GetTickCount();
hThread = CreateThread(NULL, STACK_SIZE,
_thread_func,
(void *) ¶, 0UL, &dwThreadID);
if (hThread == 0) {
HTInfoMsg("CreateThread Failed (read)");
goto read_exit;
}
i = 0;
while (1) {
val = WaitForSingleObject(hThread, 1000 / TICK);
i++;
if (val == WAIT_FAILED) {
HTInfoMsg("Wait Failed");
ret_val = DWORD_ERR;
break;
} else if (val == WAIT_TIMEOUT) {
i++;
if (i / TICK > (AlertSecs + 2)) {
sprintf(buff, "Read Waiting (%2d.%01d) for %d Bytes",
i / TICK, (i % TICK) * 10 / TICK, len);
SetConsoleTitle(buff);
}
if (win32_check_interrupt() || ((i / TICK) > lynx_timeout)) {
if (CloseHandle(hThread) == FALSE) {
HTInfoMsg("Thread terminate Failed");
}
WSASetLastError(ETIMEDOUT);
ret_val = HT_INTERRUPTED;
break;
}
} else if (val == WAIT_OBJECT_0) {
if (GetExitCodeThread(hThread, &exitcode) == FALSE) {
exitcode = DWORD_ERR;
}
if (CloseHandle(hThread) == FALSE) {
HTInfoMsg("Thread terminate Failed");
}
now_TickCount = GetTickCount();
if (now_TickCount >= save_TickCount)
process_time = now_TickCount - save_TickCount;
else
process_time = now_TickCount + (0xffffffff - save_TickCount);
if (process_time == 0)
process_time = 1;
g_total_times += process_time;
/*
* DWORD is unsigned, and could be an error code which is signed.
*/
if ((long) exitcode > 0)
g_total_bytes += exitcode;
ws_read_per_sec = g_total_bytes;
if (ws_read_per_sec > 2000000) {
if (g_total_times > 1000)
ws_read_per_sec /= (g_total_times / 1000);
} else {
ws_read_per_sec *= 1000;
ws_read_per_sec /= g_total_times;
}
ret_val = exitcode;
break;
}
} /* end while(1) */
read_exit:
LeaveCriticalSection(&critSec_READ);
return ret_val;
}
#endif /* _WINDOWS */
/*
* RFC-1738 says we can have user/password using these ASCII characters
* safe = "$" | "-" | "_" | "." | "+"
* extra = "!" | "*" | "'" | "(" | ")" | ","
* hex = digit | "A" | "B" | "C" | "D" | "E" | "F" |
* "a" | "b" | "c" | "d" | "e" | "f"
* escape = "%" hex hex
* unreserved = alpha | digit | safe | extra
* uchar = unreserved | escape
* user = *[ uchar | ";" | "?" | "&" | "=" ]
* password = *[ uchar | ";" | "?" | "&" | "=" ]
* and we cannot have a password without user, i.e., no leading ":"
* and ":", "@", "/" must be encoded, i.e., will not appear as such.
*
* However, in a URL
* //<user>:<password>@<host>:<port>/<url-path>
* valid characters in the host are different, not allowing most of those
* punctuation characters.
*
* RFC-3986 amends this, using
* userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
* unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
* reserved = gen-delims / sub-delims
* gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
* sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
* / "*" / "+" / "," / ";" / "="
* and
* host = IP-literal / IPv4address / reg-name
* reg-name = *( unreserved / pct-encoded / sub-delims )
*/
char *HTSkipToAt(char *host, int *gen_delims)
{
char *result = 0;
char *s = host;
int pass = 0;
int ch;
int last = -1;
*gen_delims = 0;
while ((ch = UCH(*s)) != '\0') {
if (ch == ':') {
if (pass++)
break;
} else if (ch == '@') {
if (s != host && last != ':')
result = s;
break;
} else if (RFC_3986_GEN_DELIMS(ch)) {
*gen_delims += 1;
if (!RFC_3986_GEN_DELIMS(s[1]))
break;
} else if (ch == '%') {
if (!(isxdigit(UCH(s[1])) && isxdigit(UCH(s[2]))))
break;
} else if (!(RFC_3986_UNRESERVED(ch) ||
RFC_3986_SUB_DELIMS(ch))) {
break;
}
++s;
last = ch;
}
return result;
}
static char *fake_hostname(char *auth)
{
char *result = NULL;
char *colon = NULL;
StrAllocCopy(result, auth);
if ((colon = strchr(result, ':')) != 0)
*colon = '\0';
if (strchr(result, '.') == 0)
FREE(result);
return result;
}
/*
* Strip any username from the given string so we retain only the host.
*/
void strip_userid(char *host, int parse_only)
{
int gen_delims = 0;
char *p1 = host;
char *p2 = HTSkipToAt(host, &gen_delims);
if (p2 != 0) {
char *msg = NULL;
char *auth = NULL;
char *fake = NULL;
char *p3 = p2;
int sub_delims = 0;
int my_delimit = UCH(*p2);
int do_trimming = (my_delimit == '@');
*p2++ = '\0';
StrAllocCopy(auth, host);
/*
* Trailing "gen-delims" demonstrates that there is no user/password.
*/
while ((p3 != host) && RFC_3986_GEN_DELIMS(p3[-1])) {
*(--p3) = '\0';
}
/*
* While legal, punctuation-only user/password is questionable.
*/
while ((p3 != host) && RFC_3986_SUB_DELIMS(p3[-1])) {
++sub_delims;
*(--p3) = '\0';
}
/*
* Trim trailing "gen-delims" from the real hostname.
*/
for (p3 = p2; *p3 != '\0'; ++p3) {
if (RFC_3986_GEN_DELIMS(*p3)) {
*p3 = '\0';
break;
}
}
CTRACE((tfp, "trim auth: result:`%s'\n", host));
if (gen_delims || strcmp(host, auth)) {
do_trimming = !gen_delims;
}
if (*host == '\0' && sub_delims) {
HTSprintf0(&msg,
gettext("User/password contains only punctuation: %s"),
auth);
} else if ((fake = fake_hostname(host)) != NULL) {
HTSprintf0(&msg,
gettext("User/password may be confused with hostname: '%s' (e.g, '%s')"),
auth, fake);
}
if (msg != 0 && !parse_only)
HTAlert(msg);
if (do_trimming) {
while ((*p1++ = *p2++) != '\0') {
;
}
CTRACE((tfp, "trim host: result:`%s'\n", host));
}
FREE(fake);
FREE(auth);
FREE(msg);
}
}
/*
* Check if the user's options specified to use the given encoding. Normally
* all encodings with compiled-in support are specified (encodingALL).
*/
static BOOL acceptEncoding(int code)
{
BOOL result = FALSE;
if ((code & LYAcceptEncoding) != 0) {
const char *program = 0;
switch (code) {
case encodingGZIP:
program = HTGetProgramPath(ppGZIP);
break;
case encodingDEFLATE:
program = HTGetProgramPath(ppINFLATE);
break;
case encodingCOMPRESS:
program = HTGetProgramPath(ppCOMPRESS);
break;
case encodingBZIP2:
program = HTGetProgramPath(ppBZIP2);
break;
default:
break;
}
/*
* FIXME: if lynx did not rely upon external programs to decompress
* files for external viewers, this check could be relaxed.
*/
result = (BOOL) (program != 0);
}
return result;
}
#ifdef USE_SSL
static void show_cert_issuer(X509 * peer_cert GCC_UNUSED)
{
#if defined(USE_OPENSSL_INCL) || defined(USE_GNUTLS_FUNCS)
char ssl_dn[1024];
char *msg = NULL;
X509_NAME_oneline(X509_get_issuer_name(peer_cert), ssl_dn, (int) sizeof(ssl_dn));
HTSprintf0(&msg, gettext("Certificate issued by: %s"), ssl_dn);
_HTProgress(msg);
FREE(msg);
#elif defined(USE_GNUTLS_INCL)
/* the OpenSSL "compat" code compiles but dumps core with GNU TLS */
#endif
}
#endif
/*
* Remove IPv6 brackets (and any port-number) from the given host-string.
*/
#ifdef USE_SSL
static char *StripIpv6Brackets(char *host)
{
int port_number;
char *p;
if ((p = HTParsePort(host, &port_number)) != 0)
*p = '\0';
if (*host == '[') {
p = host + strlen(host) - 1;
if (*p == ']') {
*p = '\0';
++host;
}
}
return host;
}
#endif
/* Load Document from HTTP Server HTLoadHTTP()
* ==============================
*
* Given a hypertext address, this routine loads a document.
*
*
* On entry,
* arg is the hypertext reference of the article to be loaded.
*
* On exit,
* returns >=0 If no error, a good socket number
* <0 Error.
*
* The socket must be closed by the caller after the document has been
* read.
*
*/
static int HTLoadHTTP(const char *arg,
HTParentAnchor *anAnchor,
HTFormat format_out,
HTStream *sink)
{
static char empty[1];
int s; /* Socket number for returned data */
const char *url = arg; /* The URL which get_physical() returned */
bstring *command = NULL; /* The whole command */
char *eol; /* End of line if found */
char *start_of_data; /* Start of body of reply */
int status; /* tcp return */
off_t bytes_already_read;
char crlf[3]; /* A CR LF equivalent string */
HTStream *target; /* Unconverted data */
HTFormat format_in; /* Format arriving in the message */
BOOL do_head = FALSE; /* Whether or not we should do a head */
BOOL do_post = FALSE; /* ARE WE posting ? */
const char *METHOD;
char *line_buffer = NULL;
char *line_kept_clean = NULL;
#ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
int real_length_of_line = 0;
#endif
BOOL extensions; /* Assume good HTTP server */
char *linebuf = NULL;
char temp[80];
BOOL first_Accept = TRUE;
BOOL show_401 = FALSE;
BOOL show_407 = FALSE;
BOOL auth_proxy = NO; /* Generate a proxy authorization. - AJL */
int length, rawlength, rv;
int server_status = 0;
BOOL doing_redirect, already_retrying = FALSE;
int len = 0;
#ifdef USE_SSL
unsigned long SSLerror;
BOOL do_connect = FALSE; /* ARE WE going to use a proxy tunnel ? */
BOOL did_connect = FALSE; /* ARE WE actually using a proxy tunnel ? */
const char *connect_url = NULL; /* The URL being proxied */
char *connect_host = NULL; /* The host being proxied */
SSL *handle = NULL; /* The SSL handle */
X509 *peer_cert; /* The peer certificate */
char ssl_dn[1024];
char *cert_host;
char *ssl_host;
char *p;
char *msg = NULL;
int status_sslcertcheck;
char *ssl_dn_start;
char *ssl_all_cns = NULL;
#ifdef USE_GNUTLS_INCL
int ret;
unsigned tls_status;
#endif
#if (SSLEAY_VERSION_NUMBER >= 0x0900) && !defined(USE_GNUTLS_FUNCS)
BOOL try_tls = TRUE;
#endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
SSL_handle = NULL;
#else
void *handle = NULL;
#endif /* USE_SSL */
if (anAnchor->isHEAD)
do_head = TRUE;
else if (anAnchor->post_data)
do_post = TRUE;
if (!url) {
status = -3;
_HTProgress(BAD_REQUEST);
goto done;
}
if (!*url) {
status = -2;
_HTProgress(BAD_REQUEST);
goto done;
}
#ifdef USE_SSL
if (using_proxy && !StrNCmp(url, "http://", 7)) {
int portnumber;
if ((connect_url = strstr((url + 7), "https://"))) {
do_connect = TRUE;
connect_host = HTParse(connect_url, "https", PARSE_HOST);
if (!HTParsePort(connect_host, &portnumber)) {
sprintf(temp, ":%d", HTTPS_PORT);
StrAllocCat(connect_host, temp);
}
CTRACE((tfp, "HTTP: connect_url = '%s'\n", connect_url));
CTRACE((tfp, "HTTP: connect_host = '%s'\n", connect_host));
} else if ((connect_url = strstr((url + 7), "snews://"))) {
do_connect = TRUE;
connect_host = HTParse(connect_url, "snews", PARSE_HOST);
if (!HTParsePort(connect_host, &portnumber)) {
sprintf(temp, ":%d", SNEWS_PORT);
StrAllocCat(connect_host, temp);
}
CTRACE((tfp, "HTTP: connect_url = '%s'\n", connect_url));
CTRACE((tfp, "HTTP: connect_host = '%s'\n", connect_host));
}
}
#endif /* USE_SSL */
sprintf(crlf, "%c%c", CR, LF);
/*
* At this point, we're talking HTTP/1.0.
*/
extensions = YES;
try_again:
/*
* All initializations are moved down here from up above, so we can start
* over here...
*/
eol = 0;
length = 0;
doing_redirect = FALSE;
permanent_redirection = FALSE;
redirect_post_content = FALSE;
target = NULL;
line_buffer = NULL;
line_kept_clean = NULL;
#ifdef USE_SSL
if (!StrNCmp(url, "https", 5))
status = HTDoConnect(url, "HTTPS", HTTPS_PORT, &s);
else
status = HTDoConnect(url, "HTTP", HTTP_PORT, &s);
#else
if (!StrNCmp(url, "https", 5)) {
HTAlert(gettext("This client does not contain support for HTTPS URLs."));
status = HT_NOT_LOADED;
goto done;
}
status = HTDoConnect(arg, "HTTP", HTTP_PORT, &s);
#endif /* USE_SSL */
if (status == HT_INTERRUPTED) {
/*
* Interrupt cleanly.
*/
CTRACE((tfp, "HTTP: Interrupted on connect; recovering cleanly.\n"));
_HTProgress(CONNECTION_INTERRUPTED);
status = HT_NOT_LOADED;
goto done;
}
if (status < 0) {
#ifdef _WINDOWS
CTRACE((tfp, "HTTP: Unable to connect to remote host for `%s'\n"
" (status = %d, sock_errno = %d).\n",
url, status, SOCKET_ERRNO));
#else
CTRACE((tfp,
"HTTP: Unable to connect to remote host for `%s' (errno = %d).\n",
url, SOCKET_ERRNO));
#endif
HTAlert(gettext("Unable to connect to remote host."));
status = HT_NOT_LOADED;
goto done;
}
#ifdef USE_SSL
use_tunnel:
/*
* If this is an https document, then do the SSL stuff here.
*/
if (did_connect || !StrNCmp(url, "https", 5)) {
SSL_handle = handle = HTGetSSLHandle();
SSL_set_fd(handle, s);
/* get host we're connecting to */
ssl_host = HTParse(url, "", PARSE_HOST);
ssl_host = StripIpv6Brackets(ssl_host);
#if defined(USE_GNUTLS_FUNCS)
ret = gnutls_server_name_set(handle->gnutls_state,
GNUTLS_NAME_DNS,
ssl_host, strlen(ssl_host));
CTRACE((tfp, "...called gnutls_server_name_set(%s) ->%d\n", ssl_host, ret));
#elif SSLEAY_VERSION_NUMBER >= 0x0900
#ifndef USE_NSS_COMPAT_INCL
if (!try_tls) {
SSL_set_no_TLSV1();
CTRACE((tfp, "...adding SSL_OP_NO_TLSv1\n"));
}
#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
else {
int ret = (int) SSL_set_tlsext_host_name(handle, ssl_host);
CTRACE((tfp, "...called SSL_set_tlsext_host_name(%s) ->%d\n",
ssl_host, ret));
}
#endif
#endif
#endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
HTSSLInitPRNG();
status = SSL_connect(handle);
if (status <= 0) {
#if (SSLEAY_VERSION_NUMBER >= 0x0900)
#if !defined(USE_GNUTLS_FUNCS)
if (try_tls) {
_HTProgress(gettext("Retrying connection without TLS."));
try_tls = FALSE;
if (did_connect)
HTTP_NETCLOSE(s, handle);
goto try_again;
} else
#endif
{
CTRACE((tfp,
"HTTP: Unable to complete SSL handshake for '%s', SSL_connect=%d, SSL error stack dump follows\n",
url, status));
SSL_load_error_strings();
while ((SSLerror = ERR_get_error()) != 0) {
CTRACE((tfp, "HTTP: SSL: %s\n", ERR_error_string(SSLerror, NULL)));
}
HTAlert("Unable to make secure connection to remote host.");
if (did_connect)
HTTP_NETCLOSE(s, handle);
status = HT_NOT_LOADED;
goto done;
}
#else
unsigned long SSLerror;
CTRACE((tfp,
"HTTP: Unable to complete SSL handshake for '%s', SSL_connect=%d, SSL error stack dump follows\n",
url, status));
SSL_load_error_strings();
while ((SSLerror = ERR_get_error()) != 0) {
CTRACE((tfp, "HTTP: SSL: %s\n", ERR_error_string(SSLerror, NULL)));
}
HTAlert("Unable to make secure connection to remote host.");
if (did_connect)
HTTP_NETCLOSE(s, handle);
status = HT_NOT_LOADED;
goto done;
#endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
}
#ifdef USE_GNUTLS_INCL
gnutls_certificate_set_verify_flags(handle->gnutls_cred,
GNUTLS_VERIFY_DO_NOT_ALLOW_SAME |
GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
ret = gnutls_certificate_verify_peers2(handle->gnutls_state, &tls_status);
if (ret < 0 || tls_status != 0) {
int flag_continue = 1;
#if GNUTLS_VERSION_NUMBER >= 0x030104
int type;
gnutls_datum_t out;
if (ret < 0) {
SSL_single_prompt(&msg,
gettext("GnuTLS error when trying to verify certificate."));
} else {
type = gnutls_certificate_type_get(handle->gnutls_state);
(void) gnutls_certificate_verification_status_print(tls_status,
type,
&out, 0);
SSL_single_prompt(&msg, (const char *) out.data);
gnutls_free(out.data);
}
#else
char *msg2;
if (ret == 0 && tls_status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
msg2 = gettext("the certificate has no known issuer");
} else if (tls_status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
msg2 = gettext("no issuer was found");
} else if (tls_status & GNUTLS_CERT_SIGNER_NOT_CA) {
msg2 = gettext("issuer is not a CA");
} else if (tls_status & GNUTLS_CERT_REVOKED) {
msg2 = gettext("the certificate has been revoked");
} else {
msg2 = gettext("the certificate is not trusted");
}
SSL_single_prompt(&msg, msg2);
#endif
CTRACE((tfp, "HTLoadHTTP: %s\n", msg));
if (!ssl_noprompt) {
if (!HTForcedPrompt(ssl_noprompt, msg, NO)) {
flag_continue = 0;
}
} else if (ssl_noprompt == FORCE_PROMPT_NO) {
flag_continue = 0;
}
FREE(msg);
if (flag_continue == 0) {
status = HT_NOT_LOADED;
FREE(msg);
goto done;
}
}
#endif
peer_cert = (X509 *) SSL_get_peer_certificate(handle);
#if defined(USE_OPENSSL_INCL) || defined(USE_GNUTLS_FUNCS)
X509_NAME_oneline(X509_get_subject_name(peer_cert),
ssl_dn, (int) sizeof(ssl_dn));
#elif defined(USE_GNUTLS_INCL)
X509_NAME_oneline(X509_get_subject_name(peer_cert),
ssl_dn + 1, (int) sizeof(ssl_dn) - 1);
/* Iterate over DN in incompatible GnuTLS format to bring it into OpenSSL format */
ssl_dn[0] = '/';
ssl_dn_start = ssl_dn;
while (*ssl_dn_start) {
if ((*ssl_dn_start == ',') && (*(ssl_dn_start + 1) == ' ')) {
*ssl_dn_start++ = '/';
if (*(p = ssl_dn_start) != 0) {
while ((p[0] = p[1]) != 0)
++p;
}
} else {
ssl_dn_start++;
}
}
#endif
/*
* X.509 DN validation taking ALL CN fields into account
* (c) 2006 Thorsten Glaser <tg@mirbsd.de>
*/
/* initialise status information */
status_sslcertcheck = 0; /* 0 = no CN found in DN */
ssl_dn_start = ssl_dn;
/* validate all CNs found in DN */
CTRACE((tfp, "Validating CNs in '%s'\n", ssl_dn_start));
while ((cert_host = strstr(ssl_dn_start, "/CN=")) != NULL) {
status_sslcertcheck = 1; /* 1 = could not verify CN */
/* start of CommonName */
cert_host += 4;
/* find next part of DistinguishedName */
if ((p = StrChr(cert_host, '/')) != NULL) {
*p = '\0';
ssl_dn_start = p; /* yes this points to the NUL byte */
} else
ssl_dn_start = NULL;
cert_host = StripIpv6Brackets(cert_host);
/* verify this CN */
CTRACE((tfp, "Matching\n\tssl_host '%s'\n\tcert_host '%s'\n",
ssl_host, cert_host));
if (!strcasecomp_asterisk(ssl_host, cert_host)) {
status_sslcertcheck = 2; /* 2 = verified peer */
/* I think this is cool to have in the logs -TG */
HTSprintf0(&msg,
gettext("Verified connection to %s (cert=%s)"),
ssl_host, cert_host);
_HTProgress(msg);
FREE(msg);
/* no need to continue the verification loop */
break;
}
/* add this CN to list of failed CNs */
if (ssl_all_cns == NULL)
StrAllocCopy(ssl_all_cns, "CN<");
else
StrAllocCat(ssl_all_cns, ":CN<");
StrAllocCat(ssl_all_cns, cert_host);
StrAllocCat(ssl_all_cns, ">");
/* if we cannot retry, don't try it */
if (ssl_dn_start == NULL)
break;
/* now retry next CN found in DN */
*ssl_dn_start = '/'; /* formerly NUL byte */
}
/* check the X.509v3 Subject Alternative Name */
#ifdef USE_GNUTLS_INCL
if (status_sslcertcheck < 2) {
int i;
size_t size;
gnutls_x509_crt_t cert;
static char buf[2048];
/* import the certificate to the x509_crt format */
if (gnutls_x509_crt_init(&cert) == 0) {
if (gnutls_x509_crt_import(cert, peer_cert,
GNUTLS_X509_FMT_DER) < 0) {
gnutls_x509_crt_deinit(cert);
goto done;
}
ret = 0;
for (i = 0; !(ret < 0); i++) {
size = sizeof(buf);
ret = gnutls_x509_crt_get_subject_alt_name(cert,
(unsigned) i,
buf, &size,
NULL);
if (strcasecomp_asterisk(ssl_host, buf) == 0) {
status_sslcertcheck = 2;
HTSprintf0(&msg,
gettext("Verified connection to %s (subj=%s)"),
ssl_host, buf);
_HTProgress(msg);
FREE(msg);
break;
}
}
}
}
#endif
#ifdef USE_OPENSSL_INCL
if (status_sslcertcheck < 2) {
STACK_OF(GENERAL_NAME) * gens;
int i, numalts;
const GENERAL_NAME *gn;
gens = (STACK_OF(GENERAL_NAME) *)
X509_get_ext_d2i(peer_cert, NID_subject_alt_name, NULL, NULL);
if (gens != NULL) {
numalts = sk_GENERAL_NAME_num(gens);
for (i = 0; i < numalts; ++i) {
gn = sk_GENERAL_NAME_value(gens, i);
if (gn->type == GEN_DNS)
cert_host = (char *) ASN1_STRING_data(gn->d.ia5);
else if (gn->type == GEN_IPADD) {
/* XXX untested -TG */
size_t j = (size_t) ASN1_STRING_length(gn->d.ia5);
cert_host = (char *) malloc(j + 1);
MemCpy(cert_host, ASN1_STRING_data(gn->d.ia5), j);
cert_host[j] = '\0';
} else
continue;
status_sslcertcheck = 1; /* got at least one */
/* verify this SubjectAltName (see above) */
cert_host = StripIpv6Brackets(cert_host);
if (!(gn->type == GEN_IPADD ? strcasecomp :
strcasecomp_asterisk) (ssl_host, cert_host)) {
status_sslcertcheck = 2;
HTSprintf0(&msg,
gettext("Verified connection to %s (subj=%s)"),
ssl_host, cert_host);
_HTProgress(msg);
FREE(msg);
if (gn->type == GEN_IPADD)
free(cert_host);
break;
}
/* add to list of failed CNs */
if (ssl_all_cns == NULL)
StrAllocCopy(ssl_all_cns, "SAN<");
else
StrAllocCat(ssl_all_cns, ":SAN<");
if (gn->type == GEN_DNS)
StrAllocCat(ssl_all_cns, "DNS=");
else if (gn->type == GEN_IPADD)
StrAllocCat(ssl_all_cns, "IP=");
StrAllocCat(ssl_all_cns, cert_host);
StrAllocCat(ssl_all_cns, ">");
if (gn->type == GEN_IPADD)
free(cert_host);
}
sk_GENERAL_NAME_free(gens);
}
}
#endif /* USE_OPENSSL_INCL */
/* if an error occurred, format the appropriate message */
if (status_sslcertcheck == 0) {
SSL_single_prompt(&msg,
gettext("Can't find common name in certificate"));
} else if (status_sslcertcheck == 1) {
SSL_double_prompt(&msg,
gettext("SSL error:host(%s)!=cert(%s)-Continue?"),
ssl_host, ssl_all_cns);
}
/* if an error occurred, let the user decide how much he trusts */
if (status_sslcertcheck < 2) {
if (msg == NULL)
StrAllocCopy(msg, gettext("SSL error"));
if (!HTForcedPrompt(ssl_noprompt, msg, NO)) {
status = HT_NOT_LOADED;
FREE(msg);
FREE(ssl_all_cns);
goto done;
}
SSL_double_prompt(&msg,
gettext("UNVERIFIED connection to %s (cert=%s)"),
ssl_host, ssl_all_cns ? ssl_all_cns : "NONE");
_HTProgress(msg);
FREE(msg);
}
show_cert_issuer(peer_cert);
HTSprintf0(&msg,
gettext("Secure %d-bit %s (%s) HTTP connection"),
SSL_get_cipher_bits(handle, NULL),
SSL_get_cipher_version(handle),
SSL_get_cipher(handle));
_HTProgress(msg);
FREE(msg);
FREE(ssl_all_cns);
FREE(ssl_host);
}
#endif /* USE_SSL */
/* Ask that node for the document, omitting the host name & anchor
*/
{
char *p1 = (HTParse(url, "", PARSE_PATH | PARSE_PUNCTUATION));
#ifdef USE_SSL
if (do_connect) {
METHOD = "CONNECT";
BStrCopy0(command, "CONNECT ");
} else
#endif /* USE_SSL */
if (do_post) {
METHOD = "POST";
BStrCopy0(command, "POST ");
} else if (do_head) {
METHOD = "HEAD";
BStrCopy0(command, "HEAD ");
} else {
METHOD = "GET";
BStrCopy0(command, "GET ");
}
/*
* If we are using a proxy gateway don't copy in the first slash of
* say: /gopher://a;lkdjfl;ajdf;lkj/;aldk/adflj so that just
* gopher://.... is sent.
*/
#ifdef USE_SSL
if (using_proxy && !did_connect) {
if (do_connect)
BStrCat0(command, connect_host);
else
BStrCat0(command, p1 + 1);
}
#else
if (using_proxy)
BStrCat0(command, p1 + 1);
#endif /* USE_SSL */
else
BStrCat0(command, p1);
FREE(p1);
}
if (extensions) {
BStrCat0(command, " ");
BStrCat0(command, ((HTprotocolLevel == HTTP_1_0)
? "HTTP/1.0"
: "HTTP/1.1"));
}
BStrCat0(command, crlf); /* CR LF, as in rfc 977 */
if (extensions) {
int n, i;
char *host = NULL;
if ((host = HTParse(anAnchor->address, "", PARSE_HOST)) != NULL) {
strip_userid(host, TRUE);
HTBprintf(&command, "Host: %s%c%c", host, CR, LF);
FREE(host);
}
if (HTprotocolLevel >= HTTP_1_1) {
HTBprintf(&command, "Connection: close%c%c", CR, LF);
}
if (!HTPresentations)
HTFormatInit();
n = HTList_count(HTPresentations);
first_Accept = TRUE;
len = 0;
for (i = 0; i < n; i++) {
HTPresentation *pres =
(HTPresentation *) HTList_objectAt(HTPresentations, i);
if (pres->get_accept) {
if (pres->quality < 1.0) {
if (pres->maxbytes > 0) {
sprintf(temp, ";q=%4.3f;mxb=%" PRI_off_t "",
pres->quality, CAST_off_t (pres->maxbytes));
} else {
sprintf(temp, ";q=%4.3f", pres->quality);
}
} else if (pres->maxbytes > 0) {
sprintf(temp, ";mxb=%" PRI_off_t "", CAST_off_t (pres->maxbytes));
} else {
temp[0] = '\0';
}
HTSprintf0(&linebuf, "%s%s%s",
(first_Accept ?
"Accept: " : ", "),
HTAtom_name(pres->rep),
temp);
len += (int) strlen(linebuf);
if (len > 252 && !first_Accept) {
BStrCat0(command, crlf);
HTSprintf0(&linebuf, "Accept: %s%s",
HTAtom_name(pres->rep),
temp);
len = (int) strlen(linebuf);
}
BStrCat0(command, linebuf);
first_Accept = FALSE;
}
}
HTBprintf(&command, "%s*/*;q=0.01%c%c",
(first_Accept ?
"Accept: " : ", "), CR, LF);
/*
* FIXME: suppressing the "Accept-Encoding" in this case is done to
* work around limitations of the presentation logic used for the
* command-line "-base" option. The remote site may transmit the
* document gzip'd, but the ensuing logic in HTSaveToFile() would see
* the mime-type as gzip rather than text/html, and not prepend the
* base URL. This is less efficient than accepting the compressed data
* and uncompressing it, adding the base URL but is simpler than
* augmenting the dump's presentation logic -TD
*/
if (LYPrependBaseToSource && dump_output_immediately) {
CTRACE((tfp,
"omit Accept-Encoding to work-around interaction with -source\n"));
} else {
char *list = 0;
int j, k;
for (j = 1; j < encodingALL; j <<= 1) {
if (acceptEncoding(j)) {
for (k = 0; tbl_preferred_encoding[k].name != 0; ++k) {
if (tbl_preferred_encoding[k].value == j) {
if (list != 0)
StrAllocCat(list, ", ");
StrAllocCat(list, tbl_preferred_encoding[k].name);
break;
}
}
}
}
if (list != 0) {
HTBprintf(&command, "Accept-Encoding: %s%c%c", list, CR, LF);
free(list);
}
}
if (non_empty(language)) {
HTBprintf(&command, "Accept-Language: %s%c%c", language, CR, LF);
}
if (non_empty(pref_charset)) {
BStrCat0(command, "Accept-Charset: ");
StrAllocCopy(linebuf, pref_charset);
if (linebuf[strlen(linebuf) - 1] == ',')
linebuf[strlen(linebuf) - 1] = '\0';
LYLowerCase(linebuf);
if (strstr(linebuf, "iso-8859-1") == NULL)
StrAllocCat(linebuf, ", iso-8859-1;q=0.01");
if (strstr(linebuf, "us-ascii") == NULL)
StrAllocCat(linebuf, ", us-ascii;q=0.01");
BStrCat0(command, linebuf);
HTBprintf(&command, "%c%c", CR, LF);
}
#if 0
/*
* Promote 300 (Multiple Choices) replies, if supported, over 406 (Not
* Acceptable) replies. - FM
*
* This used to be done in versions 2.7 and 2.8*, but violates the
* specs for transparent content negotiation and has the effect that
* servers supporting those specs will send 300 (Multiple Choices)
* instead of a normal response (e.g. 200 OK), since they will assume
* that the client wants to make the choice. It is not clear whether
* there are any servers or sites for which sending this header really
* improves anything.
*
* If there ever is a need to send "Negotiate: trans" and really mean
* it, we should send "Negotiate: trans,trans" or similar, since that
* is semantically equivalent and some servers may ignore "Negotiate:
* trans" as a special case when it comes from Lynx (to work around the
* old faulty behavior). - kw
*
* References:
* RFC 2295 (see also RFC 2296), and mail to lynx-dev and
* new-httpd@apache.org from Koen Holtman, Jan 1999.
*/
if (!do_post) {
HTBprintf(&command, "Negotiate: trans%c%c", CR, LF);
}
#endif /* 0 */
/*
* When reloading give no-cache pragma to proxy server to make it
* refresh its cache. -- Ari L. <luotonen@dxcern.cern.ch>
*
* Also send it as a Cache-Control header for HTTP/1.1. - FM
*/
if (reloading) {
HTBprintf(&command, "Pragma: no-cache%c%c", CR, LF);
HTBprintf(&command, "Cache-Control: no-cache%c%c", CR, LF);
}
if (LYSendUserAgent || no_useragent) {
if (non_empty(LYUserAgent)) {
char *cp = LYSkipBlanks(LYUserAgent);
/* Won't send it at all if all blank - kw */
if (*cp != '\0')
HTBprintf(&command, "User-Agent: %.*s%c%c",
INIT_LINE_SIZE - 15, LYUserAgent, CR, LF);
} else {
HTBprintf(&command, "User-Agent: %s/%s libwww-FM/%s%c%c",
HTAppName ? HTAppName : "unknown",
HTAppVersion ? HTAppVersion : "0.0",
HTLibraryVersion, CR, LF);
}
}
if (non_empty(personal_mail_address) && !LYNoFromHeader) {
HTBprintf(&command, "From: %s%c%c", personal_mail_address, CR, LF);
}
if (!(LYUserSpecifiedURL ||
LYNoRefererHeader || LYNoRefererForThis) &&
strcmp(HTLoadedDocumentURL(), "")) {
const char *cp = LYRequestReferer;
if (!cp)
cp = HTLoadedDocumentURL(); /* @@@ Try both? - kw */
BStrCat0(command, "Referer: ");
if (isLYNXIMGMAP(cp)) {
char *pound = findPoundSelector(cp);
int nn = (pound ? (int) (pound - cp) : (int) strlen(cp));
HTSABCat(&command, cp + LEN_LYNXIMGMAP, nn);
} else {
BStrCat0(command, cp);
}
HTBprintf(&command, "%c%c", CR, LF);
} {
char *abspath;
char *docname;
char *hostname;
char *colon;
int portnumber;
char *auth, *cookie = NULL;
BOOL secure = (BOOL) (StrNCmp(anAnchor->address, "https", 5)
? FALSE
: TRUE);
abspath = HTParse(arg, "", PARSE_PATH | PARSE_PUNCTUATION);
docname = HTParse(arg, "", PARSE_PATH);
hostname = HTParse(arg, "", PARSE_HOST);
if (hostname &&
NULL != (colon = HTParsePort(hostname, &portnumber))) {
*colon = '\0'; /* Chop off port number */
} else if (!StrNCmp(arg, "https", 5)) {
portnumber = HTTPS_PORT;
} else {
portnumber = HTTP_PORT;
}
/*
* Add Authorization, Proxy-Authorization, and/or Cookie headers,
* if applicable.
*/
if (using_proxy) {
/*
* If we are using a proxy, first determine if we should
* include an Authorization header and/or Cookie header for the
* ultimate target of this request. - FM & AJL
*/
char *host2 = NULL, *path2 = NULL;
int port2 = (StrNCmp(docname, "https", 5) ?
HTTP_PORT : HTTPS_PORT);
host2 = HTParse(docname, "", PARSE_HOST);
path2 = HTParse(docname, "", PARSE_PATH | PARSE_PUNCTUATION);
if ((colon = HTParsePort(host2, &port2)) != NULL) {
/* Use non-default port number */
*colon = '\0';
}
/*
* This composeAuth() does file access, i.e., for the ultimate
* target of the request. - AJL
*/
auth_proxy = NO;
auth = HTAA_composeAuth(host2, port2, path2, auth_proxy);
if (auth == NULL) {
CTRACE((tfp, "HTTP: Not sending authorization (yet).\n"));
} else if (*auth != '\0') {
/*
* We have an Authorization header to be included.
*/
HTBprintf(&command, "%s%c%c", auth, CR, LF);
CTRACE((tfp, "HTTP: Sending authorization: %s\n", auth));
} else {
/*
* The user either cancelled or made a mistake with the
* username and password prompt.
*/
if (!(traversal || dump_output_immediately) &&
HTConfirm(CONFIRM_WO_PASSWORD)) {
show_401 = TRUE;
} else {
if (traversal || dump_output_immediately)
HTAlert(FAILED_NEED_PASSWD);
#ifdef USE_SSL
if (did_connect)
HTTP_NETCLOSE(s, handle);
#endif /* USE_SSL */
BStrFree(command);
FREE(hostname);
FREE(docname);
FREE(abspath);
FREE(host2);
FREE(path2);
status = HT_NOT_LOADED;
goto done;
}
}
/*
* Add 'Cookie:' header, if it's HTTP or HTTPS document being
* proxied.
*/
if (!StrNCmp(docname, "http", 4)) {
cookie = LYAddCookieHeader(host2, path2, port2, secure);
}
FREE(host2);
FREE(path2);
/*
* The next composeAuth() will be for the proxy. - AJL
*/
auth_proxy = YES;
} else {
/*
* Add cookie for a non-proxied request. - FM
*/
cookie = LYAddCookieHeader(hostname, abspath, portnumber, secure);
auth_proxy = NO;
}
/*
* If we do have a cookie set, add it to the request buffer. - FM
*/
if (cookie != NULL) {
if (*cookie != '$') {
/*
* It's a historical cookie, so signal to the server that
* we support modern cookies. - FM
*/
BStrCat0(command, "Cookie2: $Version=\"1\"");
BStrCat0(command, crlf);
CTRACE((tfp, "HTTP: Sending Cookie2: $Version =\"1\"\n"));
}
if (*cookie != '\0') {
/*
* It's not a zero-length string, so add the header. Note
* that any folding of long strings has been done already
* in LYCookie.c. - FM
*/
BStrCat0(command, "Cookie: ");
BStrCat0(command, cookie);
BStrCat0(command, crlf);
CTRACE((tfp, "HTTP: Sending Cookie: %s\n", cookie));
}
FREE(cookie);
}
FREE(abspath);
/*
* If we are using a proxy, auth_proxy should be YES, and we check
* here whether we want a Proxy-Authorization header for it. If we
* are not using a proxy, auth_proxy should still be NO, and we
* check here for whether we want an Authorization header. - FM &
* AJL
*/
if ((auth = HTAA_composeAuth(hostname,
portnumber,
docname,
auth_proxy)) != NULL &&
*auth != '\0') {
/*
* If auth is not NULL nor zero-length, it's an Authorization
* or Proxy-Authorization header to be included. - FM
*/
HTBprintf(&command, "%s%c%c", auth, CR, LF);
CTRACE((tfp, (auth_proxy ?
"HTTP: Sending proxy authorization: %s\n" :
"HTTP: Sending authorization: %s\n"),
auth));
} else if (auth && *auth == '\0') {
/*
* If auth is a zero-length string, the user either cancelled
* or goofed at the username and password prompt. - FM
*/
if (!(traversal || dump_output_immediately) && HTConfirm(CONFIRM_WO_PASSWORD)) {
if (auth_proxy == TRUE) {
show_407 = TRUE;
} else {
show_401 = TRUE;
}
} else {
if (traversal || dump_output_immediately)
HTAlert(FAILED_NEED_PASSWD);
BStrFree(command);
FREE(hostname);
FREE(docname);
status = HT_NOT_LOADED;
goto done;
}
} else {
CTRACE((tfp, (auth_proxy ?
"HTTP: Not sending proxy authorization (yet).\n" :
"HTTP: Not sending authorization (yet).\n")));
}
FREE(hostname);
FREE(docname);
}
}
if (
#ifdef USE_SSL
!do_connect &&
#endif /* USE_SSL */
do_post) {
CTRACE((tfp, "HTTP: Doing post, content-type '%s'\n",
anAnchor->post_content_type
? anAnchor->post_content_type
: "lose"));
HTBprintf(&command, "Content-type: %s%c%c",
anAnchor->post_content_type
? anAnchor->post_content_type
: "lose",
CR, LF);
HTBprintf(&command, "Content-length: %d%c%c",
!isBEmpty(anAnchor->post_data)
? BStrLen(anAnchor->post_data)
: 0,
CR, LF);
BStrCat0(command, crlf); /* Blank line means "end" of headers */
BStrCat(command, anAnchor->post_data);
} else
BStrCat0(command, crlf); /* Blank line means "end" of headers */
if (TRACE) {
CTRACE((tfp, "Writing:\n"));
trace_bstring(command);
#ifdef USE_SSL
CTRACE((tfp, "%s",
(anAnchor->post_data && !do_connect ? crlf : "")));
#else
CTRACE((tfp, "%s",
(anAnchor->post_data ? crlf : "")));
#endif /* USE_SSL */
CTRACE((tfp, "----------------------------------\n"));
}
_HTProgress(gettext("Sending HTTP request."));
#ifdef NOT_ASCII /* S/390 -- gil -- 0548 */
{
char *p2;
for (p2 = BStrData(command);
p2 < BStrData(command) + BStrLen(command);
p2++)
*p2 = TOASCII(*p2);
}
#endif /* NOT_ASCII */
status = (int) HTTP_NETWRITE(s,
BStrData(command),
BStrLen(command),
handle);
BStrFree(command);
FREE(linebuf);
if (status <= 0) {
if (status == 0) {
CTRACE((tfp, "HTTP: Got status 0 in initial write\n"));
/* Do nothing. */
} else if ((SOCKET_ERRNO == ENOTCONN ||
SOCKET_ERRNO == ECONNRESET ||
SOCKET_ERRNO == EPIPE) &&
!already_retrying &&
/* Don't retry if we're posting. */ !do_post) {
/*
* Arrrrgh, HTTP 0/1 compatibility problem, maybe.
*/
CTRACE((tfp,
"HTTP: BONZO ON WRITE Trying again with HTTP0 request.\n"));
_HTProgress(RETRYING_AS_HTTP0);
HTTP_NETCLOSE(s, handle);
extensions = NO;
already_retrying = TRUE;
goto try_again;
} else {
CTRACE((tfp,
"HTTP: Hit unexpected network WRITE error; aborting connection.\n"));
HTTP_NETCLOSE(s, handle);
status = -1;
HTAlert(gettext("Unexpected network write error; connection aborted."));
goto done;
}
}
CTRACE((tfp, "HTTP: WRITE delivered OK\n"));
_HTProgress(gettext("HTTP request sent; waiting for response."));
/* Read the first line of the response
* -----------------------------------
*/
{
/* Get numeric status etc */
BOOL end_of_file = NO;
int buffer_length = INIT_LINE_SIZE;
line_buffer = typecallocn(char, (size_t) buffer_length);
if (line_buffer == NULL)
outofmem(__FILE__, "HTLoadHTTP");
HTReadProgress(bytes_already_read = 0, (off_t) 0);
do { /* Loop to read in the first line */
/*
* Extend line buffer if necessary for those crazy WAIS URLs ;-)
*/
if (buffer_length - length < LINE_EXTEND_THRESH) {
buffer_length = buffer_length + buffer_length;
line_buffer =
(char *) realloc(line_buffer, ((unsigned) buffer_length *
sizeof(char)));
if (line_buffer == NULL)
outofmem(__FILE__, "HTLoadHTTP");
}
CTRACE((tfp, "HTTP: Trying to read %d\n", buffer_length - length - 1));
status = HTTP_NETREAD(s,
line_buffer + length,
(buffer_length - length - 1),
handle);
CTRACE((tfp, "HTTP: Read %d\n", status));
if (status <= 0) {
/*
* Retry if we get nothing back too.
* Bomb out if we get nothing twice.
*/
if (status == HT_INTERRUPTED) {
CTRACE((tfp, "HTTP: Interrupted initial read.\n"));
_HTProgress(CONNECTION_INTERRUPTED);
HTTP_NETCLOSE(s, handle);
status = HT_NO_DATA;
goto clean_up;
} else if (status < 0 &&
(SOCKET_ERRNO == ENOTCONN ||
#ifdef _WINDOWS /* 1997/11/09 (Sun) 16:59:58 */
SOCKET_ERRNO == ETIMEDOUT ||
#endif
SOCKET_ERRNO == ECONNRESET ||
SOCKET_ERRNO == EPIPE) &&
!already_retrying && !do_post) {
/*
* Arrrrgh, HTTP 0/1 compability problem, maybe.
*/
CTRACE((tfp,
"HTTP: BONZO Trying again with HTTP0 request.\n"));
HTTP_NETCLOSE(s, handle);
FREE(line_buffer);
FREE(line_kept_clean);
extensions = NO;
already_retrying = TRUE;
_HTProgress(RETRYING_AS_HTTP0);
goto try_again;
}
#ifdef USE_SSL
else if ((SSLerror = ERR_get_error()) != 0) {
CTRACE((tfp,
"HTTP: Hit unexpected network read error; aborting connection; status %d:%s.\n",
status, ERR_error_string(SSLerror, NULL)));
HTAlert(gettext("Unexpected network read error; connection aborted."));
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
#endif
else {
CTRACE((tfp,
"HTTP: Hit unexpected network read error; aborting connection; status %d.\n",
status));
HTAlert(gettext("Unexpected network read error; connection aborted."));
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
}
#ifdef NOT_ASCII /* S/390 -- gil -- 0564 */
{
char *p2;
for (p2 = line_buffer + length;
p2 < line_buffer + length + status;
p2++)
*p2 = FROMASCII(*p2);
}
#endif /* NOT_ASCII */
bytes_already_read += status;
HTReadProgress(bytes_already_read, (off_t) 0);
#ifdef UCX /* UCX returns -1 on EOF */
if (status == 0 || status == -1)
#else
if (status == 0)
#endif
{
break;
}
line_buffer[length + status] = 0;
if (line_buffer) {
FREE(line_kept_clean);
line_kept_clean = (char *) malloc((unsigned) buffer_length *
sizeof(char));
if (line_kept_clean == NULL)
outofmem(__FILE__, "HTLoadHTTP");
MemCpy(line_kept_clean, line_buffer, buffer_length);
#ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
real_length_of_line = length + status;
#endif
}
eol = StrChr(line_buffer + length, LF);
/* Do we *really* want to do this? */
if (eol && eol != line_buffer && *(eol - 1) == CR)
*(eol - 1) = ' ';
length = length + status;
/* Do we really want to do *this*? */
if (eol)
*eol = 0; /* Terminate the line */
}
/* All we need is the first line of the response. If it's a HTTP/1.0
* response, then the first line will be absurdly short and therefore
* we can safely gate the number of bytes read through this code (as
* opposed to below) to ~1000.
*
* Well, let's try 100.
*/
while (!eol && !end_of_file && bytes_already_read < 100);
} /* Scope of loop variables */
/* save total length, in case we decide later to show it all - kw */
rawlength = length;
/* We now have a terminated unfolded line. Parse it.
* --------------------------------------------------
*/
CTRACE((tfp, "HTTP: Rx: %s\n", line_buffer));
/*
* Kludge to work with old buggy servers and the VMS Help gateway. They
* can't handle the third word, so we try again without it.
*/
if (extensions && /* Old buggy server or Help gateway? */
(0 == StrNCmp(line_buffer, "<TITLE>Bad File Request</TITLE>", 31) ||
0 == StrNCmp(line_buffer, "Address should begin with", 25) ||
0 == StrNCmp(line_buffer, "<TITLE>Help ", 12) ||
0 == strcmp(line_buffer,
"Document address invalid or access not authorised"))) {
FREE(line_buffer);
FREE(line_kept_clean);
extensions = NO;
already_retrying = TRUE;
CTRACE((tfp, "HTTP: close socket %d to retry with HTTP0\n", s));
HTTP_NETCLOSE(s, handle);
/* print a progress message */
_HTProgress(RETRYING_AS_HTTP0);
goto try_again;
} {
int fields;
char server_version[VERSION_LENGTH + 1];
server_version[0] = 0;
fields = sscanf(line_buffer, "%20s %d",
server_version,
&server_status);
CTRACE((tfp, "HTTP: Scanned %d fields from line_buffer\n", fields));
if (non_empty(http_error_file)) {
/* Make the status code externally available */
FILE *error_file;
#ifdef SERVER_STATUS_ONLY
error_file = fopen(http_error_file, TXT_W);
if (error_file) { /* Managed to open the file */
fprintf(error_file, "error=%d\n", server_status);
fclose(error_file);
}
#else
error_file = fopen(http_error_file, TXT_A);
if (error_file) { /* Managed to open the file */
fprintf(error_file, " URL=%s (%s)\n", url, METHOD);
fprintf(error_file, "STATUS=%s\n", line_buffer);
fclose(error_file);
}
#endif /* SERVER_STATUS_ONLY */
}
/*
* Rule out a non-HTTP/1.n reply as best we can.
*/
if (fields < 2 || !server_version[0] || server_version[0] != 'H' ||
server_version[1] != 'T' || server_version[2] != 'T' ||
server_version[3] != 'P' || server_version[4] != '/' ||
server_version[6] != '.') {
/*
* Ugh! An HTTP0 reply,
*/
HTAtom *encoding;
CTRACE((tfp, "--- Talking HTTP0.\n"));
format_in = HTFileFormat(url, &encoding, NULL);
/*
* Treat all plain text as HTML. This sucks but its the only
* solution without without looking at content.
*/
if (!StrNCmp(HTAtom_name(format_in), STR_PLAINTEXT, 10)) {
CTRACE((tfp, "HTTP: format_in being changed to text/HTML\n"));
format_in = WWW_HTML;
}
if (!IsUnityEnc(encoding)) {
/*
* Change the format to that for "www/compressed".
*/
CTRACE((tfp, "HTTP: format_in is '%s',\n", HTAtom_name(format_in)));
StrAllocCopy(anAnchor->content_type, HTAtom_name(format_in));
StrAllocCopy(anAnchor->content_encoding, HTAtom_name(encoding));
format_in = HTAtom_for("www/compressed");
CTRACE((tfp, " Treating as '%s' with encoding '%s'\n",
"www/compressed", HTAtom_name(encoding)));
}
start_of_data = line_kept_clean;
} else {
/*
* Set up to decode full HTTP/1.n response. - FM
*/
format_in = HTAtom_for("www/mime");
CTRACE((tfp, "--- Talking HTTP1.\n"));
/*
* We set start_of_data to "" when !eol here because there will be
* a put_block done below; we do *not* use the value of
* start_of_data (as a pointer) in the computation of length (or
* anything else) when !eol. Otherwise, set the value of length to
* what we have beyond eol (i.e., beyond the status line). - FM
*/
if (eol != 0) {
start_of_data = (eol + 1);
} else {
start_of_data = empty;
}
length = (eol
? length - (int) (start_of_data - line_buffer)
: 0);
/*
* Trim trailing spaces in line_buffer so that we can use it in
* messages which include the status line. - FM
*/
while (line_buffer[strlen(line_buffer) - 1] == ' ')
line_buffer[strlen(line_buffer) - 1] = '\0';
/*
* Take appropriate actions based on the status. - FM
*/
switch (server_status / 100) {
case 1:
/*
* HTTP/1.1 Informational statuses.
* 100 Continue.
* 101 Switching Protocols.
* > 101 is unknown.
* We should never get these, and they have only the status
* line and possibly other headers, so we'll deal with them by
* showing the full header to the user as text/plain. - FM
*/
HTAlert(gettext("Got unexpected Informational Status."));
do_head = TRUE;
break;
case 2:
/*
* Good: Got MIME object! (Successful) - FM
*/
if (do_head) {
/*
* If HEAD was requested, show headers (and possibly bogus
* body) for all 2xx status codes as text/plain - KW
*/
HTProgress(line_buffer);
break;
}
switch (server_status) {
case 204:
/*
* No Content.
*/
HTAlert(line_buffer);
HTTP_NETCLOSE(s, handle);
HTNoDataOK = 1;
status = HT_NO_DATA;
goto clean_up;
case 205:
/*
* Reset Content. The server has fulfilled the request but
* nothing is returned and we should reset any form
* content. We'll instruct the user to do that, and
* restore the current document. - FM
*/
HTAlert(gettext("Request fulfilled. Reset Content."));
HTTP_NETCLOSE(s, handle);
status = HT_NO_DATA;
goto clean_up;
case 206:
/*
* Partial Content. We didn't send a Range so something
* went wrong somewhere. Show the status message and
* restore the current document. - FM
*/
HTAlert(line_buffer);
HTTP_NETCLOSE(s, handle);
status = HT_NO_DATA;
goto clean_up;
default:
/*
* 200 OK.
* 201 Created.
* 202 Accepted.
* 203 Non-Authoritative Information.
* > 206 is unknown.
* All should return something to display.
*/
#if defined(USE_SSL) /* && !defined(DISABLE_NEWS) _H */
if (do_connect) {
CTRACE((tfp,
"HTTP: Proxy tunnel to '%s' established.\n",
connect_host));
do_connect = FALSE;
url = connect_url;
FREE(line_buffer);
FREE(line_kept_clean);
#ifndef DISABLE_NEWS
if (!StrNCmp(connect_url, "snews", 5)) {
CTRACE((tfp,
" Will attempt handshake and snews connection.\n"));
status = HTNewsProxyConnect(s, url, anAnchor,
format_out, sink);
goto done;
}
#endif /* DISABLE_NEWS */
did_connect = TRUE;
already_retrying = TRUE;
eol = 0;
length = 0;
doing_redirect = FALSE;
permanent_redirection = FALSE;
target = NULL;
CTRACE((tfp,
" Will attempt handshake and resubmit headers.\n"));
goto use_tunnel;
}
#endif /* USE_SSL */
HTProgress(line_buffer);
} /* case 2 switch */
break;
case 3:
/*
* Various forms of Redirection. - FM
* 300 Multiple Choices.
* 301 Moved Permanently.
* 302 Found (temporary; we can, and do, use GET).
* 303 See Other (temporary; always use GET).
* 304 Not Modified.
* 305 Use Proxy.
* 306 Set Proxy.
* 307 Temporary Redirect with method retained.
* > 308 is unknown.
*/
if (no_url_redirection || do_head || keep_mime_headers) {
/*
* If any of these flags are set, we do not redirect, but
* instead show what was returned to the user as
* text/plain. - FM
*/
HTProgress(line_buffer);
break;
}
if (server_status == 300) { /* Multiple Choices */
/*
* For client driven content negotiation. The server
* should be sending some way for the user-agent to make a
* selection, so we'll show the user whatever the server
* returns. There might be a Location: header with the
* server's preference present, but the choice should be up
* to the user, someday based on an Alternates: header,
* and a body always should be present with descriptions
* and links for the choices (i.e., we use the latter, for
* now). - FM
*/
HTAlert(line_buffer);
if (traversal) {
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request
* for interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
}
if (server_status == 304) { /* Not Modified */
/*
* We didn't send an "If-Modified-Since" header, so this
* status is inappropriate. We'll deal with it by showing
* the full header to the user as text/plain. - FM
*/
HTAlert(gettext("Got unexpected 304 Not Modified status."));
do_head = TRUE;
break;
}
if (server_status == 305 ||
server_status == 306 ||
server_status > 307) {
/*
* Show user the content, if any, for 305, 306, or unknown
* status. - FM
*/
HTAlert(line_buffer);
if (traversal) {
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request
* for interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
}
/*
* We do not load the file, but read the headers for the
* "Location:", check out that redirecting_url and if it's
* acceptible (e.g., not a telnet URL when we have that
* disabled), initiate a new fetch. If that's another
* redirecting_url, we'll repeat the checks, and fetch
* initiations if acceptible, until we reach the actual URL, or
* the redirection limit set in HTAccess.c is exceeded. If the
* status was 301 indicating that the relocation is permanent,
* we set the permanent_redirection flag to make it permanent
* for the current anchor tree (i.e., will persist until the
* tree is freed or the client exits). If the redirection
* would include POST content, we seek confirmation from an
* interactive user, with option to use 303 for 301 (but not
* for 307), and otherwise refuse the redirection. We also
* don't allow permanent redirection if we keep POST content.
* If we don't find the Location header or it's value is
* zero-length, we display whatever the server returned, and
* the user should RELOAD that to try again, or make a
* selection from it if it contains links, or Left-Arrow to the
* previous document. - FM
*/
{
if ((dump_output_immediately || traversal) &&
do_post &&
server_status != 303 &&
server_status != 302 &&
server_status != 301) {
/*
* Don't redirect POST content without approval from an
* interactive user. - FM
*/
HTTP_NETCLOSE(s, handle);
status = -1;
HTAlert(gettext("Redirection of POST content requires user approval."));
if (traversal)
HTProgress(line_buffer);
goto clean_up;
}
HTProgress(line_buffer);
if (server_status == 301) { /* Moved Permanently */
if (do_post) {
/*
* Don't make the redirection permanent if we have
* POST content. - FM
*/
CTRACE((tfp,
"HTTP: Have POST content. Treating 301 (Permanent) as Temporary.\n"));
HTAlert(gettext("Have POST content. Treating Permanent Redirection as Temporary.\n"));
} else {
permanent_redirection = TRUE;
}
}
doing_redirect = TRUE;
break;
}
case 4:
/*
* "I think I goofed!" (Client Error) - FM
*/
switch (server_status) {
case 401: /* Unauthorized */
/*
* Authorization for origin server required. If show_401
* is set, proceed to showing the 401 body. Otherwise, if
* we can set up authorization based on the
* WWW-Authenticate header, and the user provides a
* username and password, try again. Otherwise, check
* whether to show the 401 body or restore the current
* document - FM
*/
if (show_401)
break;
if (HTAA_shouldRetryWithAuth(start_of_data, (size_t)
length, s, NO)) {
HTTP_NETCLOSE(s, handle);
if (dump_output_immediately &&
!HTAA_HaveUserinfo(HTParse(arg, "", PARSE_HOST)) &&
!authentication_info[0]) {
fprintf(stderr,
"HTTP: Access authorization required.\n");
fprintf(stderr,
" Use the -auth=id:pw parameter.\n");
status = HT_NO_DATA;
goto clean_up;
}
CTRACE((tfp, "%s %d %s\n",
"HTTP: close socket", s,
"to retry with Access Authorization"));
_HTProgress(gettext("Retrying with access authorization information."));
FREE(line_buffer);
FREE(line_kept_clean);
#ifdef USE_SSL
if (using_proxy && !StrNCmp(url, "https://", 8)) {
url = arg;
do_connect = TRUE;
did_connect = FALSE;
}
#endif /* USE_SSL */
goto try_again;
} else if (!(traversal || dump_output_immediately) &&
HTConfirm(gettext("Show the 401 message body?"))) {
break;
} else {
if (traversal || dump_output_immediately)
HTAlert(FAILED_RETRY_WITH_AUTH);
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
case 407:
/*
* Authorization for proxy server required. If we are not
* in fact using a proxy, or show_407 is set, proceed to
* showing the 407 body. Otherwise, if we can set up
* authorization based on the Proxy-Authenticate header,
* and the user provides a username and password, try
* again. Otherwise, check whether to show the 401 body or
* restore the current document. - FM & AJL
*/
if (!using_proxy || show_407)
break;
if (HTAA_shouldRetryWithAuth(start_of_data, (size_t)
length, s, YES)) {
HTTP_NETCLOSE(s, handle);
if (dump_output_immediately && !proxyauth_info[0]) {
fprintf(stderr,
"HTTP: Proxy authorization required.\n");
fprintf(stderr,
" Use the -pauth=id:pw parameter.\n");
status = HT_NO_DATA;
goto clean_up;
}
CTRACE((tfp, "%s %d %s\n",
"HTTP: close socket", s,
"to retry with Proxy Authorization"));
_HTProgress(HTTP_RETRY_WITH_PROXY);
FREE(line_buffer);
FREE(line_kept_clean);
goto try_again;
} else if (!(traversal || dump_output_immediately) &&
HTConfirm(gettext("Show the 407 message body?"))) {
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation
* request for interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
} else {
if (traversal || dump_output_immediately)
HTAlert(FAILED_RETRY_WITH_PROXY);
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
case 408:
/*
* Request Timeout. Show the status message and restore
* the current document. - FM
*/
HTAlert(line_buffer);
HTTP_NETCLOSE(s, handle);
status = HT_NO_DATA;
goto clean_up;
default:
/*
* 400 Bad Request.
* 402 Payment Required.
* 403 Forbidden.
* 404 Not Found.
* 405 Method Not Allowed.
* 406 Not Acceptable.
* 409 Conflict.
* 410 Gone.
* 411 Length Required.
* 412 Precondition Failed.
* 413 Request Entity Too Large.
* 414 Request-URI Too Long.
* 415 Unsupported Media Type.
* 416 List Response (for content negotiation).
* > 416 is unknown.
* Show the status message, and display the returned text
* if we are not doing a traversal. - FM
*/
HTAlert(line_buffer);
if (traversal) {
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request
* for interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
} /* case 4 switch */
break;
case 5:
/*
* "I think YOU goofed!" (server error)
* 500 Internal Server Error
* 501 Not Implemented
* 502 Bad Gateway
* 503 Service Unavailable
* 504 Gateway Timeout
* 505 HTTP Version Not Supported
* > 505 is unknown.
* Should always include a message, which we always should
* display. - FM
*/
HTAlert(line_buffer);
if (traversal) {
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request for
* interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
default:
/*
* Bad or unknown server_status number. Take a chance and hope
* there is something to display. - FM
*/
HTAlert(gettext("Unknown status reply from server!"));
HTAlert(line_buffer);
if (traversal) {
HTTP_NETCLOSE(s, handle);
status = -1;
goto clean_up;
}
if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request for
* interactive users. - FM
*/
format_out = WWW_PRESENT;
}
break;
} /* Switch on server_status/100 */
} /* Full HTTP reply */
} /* scope of fields */
/*
* The user may have pressed the 'z'ap key during the pause caused by one
* of the HTAlerts above if the server reported an error, to skip loading
* of the error response page. Checking here before setting up the stream
* stack and feeding it data avoids doing unnecessary work, it also can
* avoid unnecessarily pushing a loaded document out of the cache to make
* room for the unwanted error page. - kw
*/
if (HTCheckForInterrupt()) {
HTTP_NETCLOSE(s, handle);
if (doing_redirect) {
/*
* Impatient user. - FM
*/
CTRACE((tfp, "HTTP: Interrupted followup read.\n"));
_HTProgress(CONNECTION_INTERRUPTED);
}
status = HT_INTERRUPTED;
goto clean_up;
}
/*
* Set up the stream stack to handle the body of the message.
*/
if (do_head || keep_mime_headers) {
/*
* It was a HEAD request, or we want the headers and source.
*/
start_of_data = line_kept_clean;
#ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
/* GIF file contains \0, so strlen does not return the data length */
length = real_length_of_line;
#else
length = rawlength;
#endif
format_in = HTAtom_for(STR_PLAINTEXT);
} else if (doing_redirect) {
format_in = HTAtom_for("message/x-http-redirection");
StrAllocCopy(anAnchor->content_type, HTAtom_name(format_in));
if (traversal) {
format_out = WWW_DEBUG;
if (!sink)
sink = HTErrorStream();
} else if (!dump_output_immediately &&
format_out == HTAtom_for("www/download")) {
/*
* Convert a download request to a presentation request for
* interactive users. - FM
*/
format_out = WWW_PRESENT;
}
}
target = HTStreamStack(format_in,
format_out,
sink, anAnchor);
if (target == NULL) {
char *buffer = NULL;
HTTP_NETCLOSE(s, handle);
HTSprintf0(&buffer, CANNOT_CONVERT_I_TO_O,
HTAtom_name(format_in), HTAtom_name(format_out));
_HTProgress(buffer);
FREE(buffer);
status = -1;
goto clean_up;
}
/*
* Recycle the first chunk of data, in all cases.
*/
(*target->isa->put_block) (target, start_of_data, length);
/*
* Go pull the bulk of the data down.
*/
rv = HTCopy(anAnchor, s, (void *) handle, target);
/*
* If we get here with doing_redirect set, it means that we were looking
* for a Location header. We either have got it now in redirecting_url -
* in that case the stream should not have loaded any data. Or we didn't
* get it, in that case the stream may have presented the message body
* normally. - kw
*/
if (rv == -1) {
/*
* Intentional interrupt before data were received, not an error
*/
if (doing_redirect && traversal)
status = -1;
else
status = HT_INTERRUPTED;
HTTP_NETCLOSE(s, handle);
goto clean_up;
}
if (rv == -2) {
/*
* Aw hell, a REAL error, maybe cuz it's a dumb HTTP0 server
*/
(*target->isa->_abort) (target, NULL);
if (doing_redirect && redirecting_url) {
/*
* Got a location before the error occurred? Then consider it an
* interrupt but proceed below as normal. - kw
*/
/* do nothing here */
} else {
HTTP_NETCLOSE(s, handle);
if (!doing_redirect && !already_retrying && !do_post) {
CTRACE((tfp, "HTTP: Trying again with HTTP0 request.\n"));
/*
* May as well consider it an interrupt -- right?
*/
FREE(line_buffer);
FREE(line_kept_clean);
extensions = NO;
already_retrying = TRUE;
_HTProgress(RETRYING_AS_HTTP0);
goto try_again;
} else {
status = HT_NOT_LOADED;
goto clean_up;
}
}
}
/*
* Free if complete transmission (socket was closed before return). Close
* socket if partial transmission (was freed on abort).
*/
if (rv != HT_INTERRUPTED && rv != -2) {
(*target->isa->_free) (target);
} else {
HTTP_NETCLOSE(s, handle);
}
if (doing_redirect) {
if (redirecting_url) {
/*
* Set up for checking redirecting_url in LYGetFile.c for
* restrictions before we seek the document at that Location. - FM
*/
CTRACE((tfp, "HTTP: Picked up location '%s'\n",
redirecting_url));
if (rv == HT_INTERRUPTED) {
/*
* Intentional interrupt after data were received, not an error
* (probably). We take it as a user request to abandon the
* redirection chain.
*
* This could reasonably be changed (by just removing this
* block), it would make sense if there are redirecting
* resources that "hang" after sending the headers. - kw
*/
FREE(redirecting_url);
CTRACE((tfp, "HTTP: Interrupted followup read.\n"));
status = HT_INTERRUPTED;
goto clean_up;
}
HTProgress(line_buffer);
if (server_status == 305) { /* Use Proxy */
/*
* Make sure the proxy field ends with a slash. - FM
*/
if (redirecting_url[strlen(redirecting_url) - 1]
!= '/')
StrAllocCat(redirecting_url, "/");
/*
* Append our URL. - FM
*/
StrAllocCat(redirecting_url, anAnchor->address);
CTRACE((tfp, "HTTP: Proxy URL is '%s'\n",
redirecting_url));
}
if (!do_post ||
server_status == 303 ||
server_status == 302) {
/*
* We don't have POST content (nor support PUT or DELETE), or
* the status is "See Other" or "General Redirection" and we
* can convert to GET, so go back and check out the new URL. -
* FM
*/
status = HT_REDIRECTING;
goto clean_up;
}
/*
* Make sure the user wants to redirect the POST content, or treat
* as GET - FM & DK
*/
switch (HTConfirmPostRedirect(redirecting_url,
server_status)) {
/*
* User failed to confirm. Abort the fetch.
*/
case 0:
FREE(redirecting_url);
status = HT_NO_DATA;
goto clean_up;
/*
* User wants to treat as GET with no content. Go back to
* check out the URL.
*/
case 303:
break;
/*
* Set the flag to retain the POST content and go back to check
* out the URL. - FM
*/
default:
redirect_post_content = TRUE;
}
/* Lou's old comment: - FM */
/* OK, now we've got the redirection URL temporarily stored
in external variable redirecting_url, exported from HTMIME.c,
since there's no straightforward way to do this in the library
currently. Do the right thing. */
status = HT_REDIRECTING;
} else {
status = traversal ? -1 : HT_LOADED;
}
} else {
/*
* If any data were received, treat as a complete transmission
*/
status = HT_LOADED;
}
/*
* Clean up
*/
clean_up:
FREE(line_buffer);
FREE(line_kept_clean);
done:
/*
* Clear out on exit, just in case.
*/
reloading = FALSE;
#ifdef USE_SSL
FREE(connect_host);
if (handle) {
SSL_free(handle);
SSL_handle = handle = NULL;
}
#endif /* USE_SSL */
dump_server_status = server_status;
return status;
}
/* Protocol descriptor
*/
#ifdef GLOBALDEF_IS_MACRO
#define _HTTP_C_GLOBALDEF_1_INIT { "http", HTLoadHTTP, 0}
GLOBALDEF(HTProtocol, HTTP, _HTTP_C_GLOBALDEF_1_INIT);
#define _HTTP_C_GLOBALDEF_2_INIT { "https", HTLoadHTTP, 0}
GLOBALDEF(HTProtocol, HTTPS, _HTTP_C_GLOBALDEF_2_INIT);
#else
GLOBALDEF HTProtocol HTTP =
{"http", HTLoadHTTP, 0};
GLOBALDEF HTProtocol HTTPS =
{"https", HTLoadHTTP, 0};
#endif /* GLOBALDEF_IS_MACRO */