/*
* http.c - dnssec-trigger HTTP client code to GET a simple URL
*
* Copyright (c) 2012, NLnet Labs. All rights reserved.
*
* This software is open source.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of the NLNET LABS nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* \file
*
* This file contains an implementation of HTTP fetch for a simple URL file.
*/
#include "config.h"
#include <ldns/ldns.h>
#include "riggerd/http.h"
#include "riggerd/netevent.h"
#include "riggerd/log.h"
#include "riggerd/svr.h"
#include "riggerd/probe.h"
#include "riggerd/cfg.h"
#include "riggerd/net_help.h"
#include "riggerd/update.h"
#ifdef USE_WINSOCK
#include "winsock_event.h"
#endif
/** start http get with a random dest address from the set */
void http_probe_start_http_get(struct http_probe* hp);
/** parse url into hostname and filename */
static int parse_url(char* url, char** h, char** f)
{
char* front, *sl;
if(strncmp(url, "http://", 7) != 0) {
return 0;
}
front = url+7;
sl = strchr(front, '/');
if(!sl) {
/* http://www.example.com */
(*h) = strdup(front);
(*f) = strdup("");
} else {
*sl = 0;
(*h) = strdup(front);
(*f) = strdup(sl+1);
*sl = '/';
}
if(!(*h) || !(*f)) {
log_err("parse_url: malloc failure");
return 0;
}
return 1;
}
/** setup httpprobe for a new url */
static int
http_probe_setup_url(struct http_general* hg, struct http_probe* hp, size_t i)
{
hp->do_addr = 1;
hp->url_idx = i;
hp->url = strdup(hg->urls[i]);
if(!hp->url) {
log_err("out of memory");
return 0;
}
if(!parse_url(hp->url, &hp->hostname, &hp->filename)) {
return 0;
}
verbose(VERB_ALGO, "setup url %s %s", hp->hostname, hp->filename);
return 1;
}
/** create probe for address */
static void
probe_create_addr(const char* ip, const char* domain, int rrtype)
{
struct probe_ip* p;
p = (struct probe_ip*)calloc(1, sizeof(*p));
if(!p) {
log_err("out of memory");
return;
}
p->port = DNS_PORT;
p->to_http = 1;
p->http_ip6 = (rrtype == LDNS_RR_TYPE_AAAA);
p->name = strdup(ip);
if(!p->name) {
free(p);
log_err("out of memory");
return;
}
/* no need for EDNS-probe, DNSSEC-types; we check for magic cookie in
* HTTP response data */
p->host_c = outq_create(p->name, rrtype, domain, 1, p, 0, 0,
p->port, 0, 0);
if(!p->host_c) {
free(p->name);
free(p);
log_err("out of memory");
return;
}
/* put it in the svr list */
p->next = global_svr->probes;
global_svr->probes = p;
global_svr->num_probes++;
}
/** create address lookup queries for http probe */
static void
http_probe_make_addr_queries(struct http_general* hg, struct http_probe* hp)
{
/* lookup hostname at the recursive resolvers */
struct probe_ip* p;
hp->num_addr_qs = 0;
hp->num_failed_addr_qs = 0;
/* created probes are prepended, thus this can continue easily */
for(p = hg->svr->probes; p; p=p->next) {
if(!probe_is_cache(p))
continue;
probe_create_addr(p->name, hp->hostname,
hp->ip6?LDNS_RR_TYPE_AAAA:LDNS_RR_TYPE_A);
hp->num_addr_qs++;
if(hp->num_addr_qs >= HTTP_MAX_ADDR_QUERIES)
break;
}
}
/** see if hp ip6 fits with probe ip6 */
static int
right_ip6(struct http_probe* hp, struct probe_ip* p)
{
return (hp->ip6 && p->http_ip6) || (!hp->ip6 && !p->http_ip6);
}
/* delete addr lookups from probe list in svr */
void http_probe_remove_addr_lookups(struct http_probe* hp)
{
struct svr* svr = global_svr;
struct probe_ip* p = svr->probes, **pp = &svr->probes;
/* find and delete addr lookups */
while(p) {
/* need to delete this? */
if(p->to_http && right_ip6(hp, p) && p->host_c) {
/* snip off */
(*pp) = p->next;
if(p->works)
svr->num_probes_done --;
svr->num_probes --;
probe_delete(p);
p = (*pp);
continue;
}
/* go to next item */
pp = &p->next;
p = p->next;
}
}
/* delete http lookups from probe list in svr */
void http_probe_remove_http_lookups(struct http_probe* hp)
{
struct svr* svr = global_svr;
struct probe_ip* p = svr->probes, **pp = &svr->probes;
verbose(VERB_ALGO, "remove http lookups");
/* find and delete http lookups */
while(p) {
/* need to delete this? */
if(p->to_http && right_ip6(hp, p) && p->http) {
/* snip off */
(*pp) = p->next;
if(p->works)
svr->num_probes_done --;
svr->num_probes --;
probe_delete(p);
p = (*pp);
continue;
}
/* go to next item */
pp = &p->next;
p = p->next;
}
}
/** parse http-hostname as IP address */
static int
parse_http_addr(char* s, struct sockaddr_storage* addr, socklen_t* len,
int* port)
{
if(s[0]=='[') {
char temp[128];
char* eb = strchr(s, ']');
/* [addr] or [addr]:port */
if(!eb) return 0; /* no close bracket */
*port = HTTP_PORT;
/* extract address part */
*eb = 0;
(void)strlcpy(temp, s+1, sizeof(temp));
*eb = ']';
/* we know that eb[0] is ']' */
if(eb[1] == 0) {
/* '[addr]' */
return ipstrtoaddr(temp, *port, addr, len);
} else if(eb[1] == ':') {
/* '[addr]:port' */
*port = atoi(eb+2);
return ipstrtoaddr(temp, *port, addr, len);
}
return 0;
}
/* attempt parse of plain ip4 or ip6 */
*port = HTTP_PORT;
if(ipstrtoaddr(s, HTTP_PORT, addr, len))
return 1;
/* attempt addr:port */
if(strchr(s, ':')) {
char temp[128];
char* e= strchr(s, ':');
/* extract address part */
*e = 0;
(void)strlcpy(temp, s, sizeof(temp));
*e = ':';
*port = atoi(e+1);
return ipstrtoaddr(temp, *port, addr, len);
}
return 0;
}
/** see if hostname is ip4, or ip6 or [ip6] or [ip6]:port or ip4:port */
static int
http_probe_hostname_has_addr(struct http_probe* hp)
{
struct sockaddr_storage addr;
socklen_t len = 0;
ldns_rr* rr;
ldns_rdf* f;
if(!parse_http_addr(hp->hostname, &addr, &len, &hp->port))
return 0;
/* setup addr structure and port */
hp->addr = ldns_rr_list_new();
if(!hp->addr) {
log_err("out of memory");
return 1;
}
/* if we are not appriopriate (IP6 addr and we are IP4) we have empty
* list as the result */
if(addr_is_ip6(&addr, len)) {
if(!hp->ip6)
return 1;
} else {
if(hp->ip6)
return 1;
}
rr = ldns_rr_new_frm_type(addr_is_ip6(&addr, len)?
LDNS_RR_TYPE_AAAA:LDNS_RR_TYPE_A);
if(!rr) {
log_err("out of memory");
return 1;
}
if(addr_is_ip6(&addr, len)) {
f = ldns_rdf_new_frm_data(LDNS_RDF_TYPE_AAAA, INET6_SIZE,
&((struct sockaddr_in6*)&addr)->sin6_addr);
} else {
f = ldns_rdf_new_frm_data(LDNS_RDF_TYPE_A, INET_SIZE,
&((struct sockaddr_in*)&addr)->sin_addr);
}
if(!f) {
ldns_rr_free(rr);
log_err("out of memory");
return 1;
}
(void)ldns_rr_a_set_address(rr, f);
if(!ldns_rr_list_push_rr(hp->addr, rr)) {
ldns_rr_free(rr);
log_err("out of memory");
return 1;
}
return 1;
}
/** the http_probe is done (fail with reason, or its is NULL) */
static void http_probe_done(struct http_general* hg,
struct http_probe* hp, char* reason)
{
hp->finished = 1;
verbose(VERB_OPS, "http probe%s %s done: %s", hp->ip6?"6":"4", hp->url,
reason?reason:"success");
if(reason == NULL) {
/* success! stop the other probe part */
if(hp->ip6 && hg->v4) {
if(hg->v4->do_addr)
http_probe_remove_addr_lookups(hg->v4);
else http_probe_remove_http_lookups(hg->v4);
} else if(!hp->ip6 && hg->v6) {
if(hg->v6->do_addr)
http_probe_remove_addr_lookups(hg->v6);
else http_probe_remove_http_lookups(hg->v6);
}
hp->works = 1;
http_general_done(reason);
} else {
hp->works = 0;
/* if other done too, now its total fail for http */
if(hp->ip6) {
if(!hg->v4 || hg->v4->finished) {
http_general_done(reason);
}
} else {
if(!hg->v6 || hg->v6->finished) {
http_general_done(reason);
}
}
}
}
/** start resolving the hostname of the next url in the list */
static void http_probe_go_next_url(struct http_general* hg,
struct http_probe* hp, char* redirect_url)
{
free(hp->url);
hp->url = NULL;
free(hp->hostname);
hp->hostname = NULL;
free(hp->filename);
hp->filename = NULL;
ldns_rr_list_deep_free(hp->addr);
hp->addr = NULL;
hp->num_addr_qs = 0;
hp->num_failed_addr_qs = 0;
hp->port = HTTP_PORT;
if(!redirect_url) {
hp->redirects = 0;
log_assert(hp->url_idx < hg->url_num);
if(!http_probe_setup_url(hg, hp, ++hp->url_idx)) {
http_probe_done(hg, hp, "out of memory or parse error");
return;
}
} else {
hp->url = redirect_url;
if(!parse_url(hp->url, &hp->hostname, &hp->filename)) {
http_probe_done(hg, hp, "out of memory or parse error");
return;
}
verbose(VERB_ALGO, "restart http probe for %s %s",
hp->hostname, hp->filename);
}
if(http_probe_hostname_has_addr(hp)) {
if(!hp->addr || ldns_rr_list_rr_count(hp->addr)==0) {
if(hp->ip6)
http_probe_done(hg, hp, "no address of type IP6");
else http_probe_done(hg, hp, "no address of type IP4");
return;
}
hp->got_addrs = 1;
hp->do_addr = 0;
http_probe_start_http_get(hp);
return;
}
http_probe_make_addr_queries(hg, hp);
}
/** http probe is done with an address, check next addr */
static void http_probe_done_addr(struct http_general* hg,
struct http_probe* hp, char* reason, int connects, char* redirect)
{
/* if we have a redirect, then we should attempt to follow this,
* if allowed to follow more redirects */
if(!reason && redirect) {
if(hp->redirects++ > HTTP_MAX_REDIRECT) {
/* redirect is strdupped and needs to be free-ed */
free(redirect);
redirect = NULL;
reason = "too many http redirects";
connects = 1;
}
}
if(!reason && redirect) {
/* re-setup the URL */
hp->do_addr = 1;
hp->got_addrs = 0;
http_probe_go_next_url(hg, hp, redirect);
return;
}
if(redirect) {
free(redirect);
redirect = NULL;
}
/* if we connected to some sort of server, then we do not need to
* attempt a different server - we are hotspotted or successed */
if(connects) {
http_probe_done(hg, hp, reason);
return;
}
if(!reason) { /* should also 'connects', but for robustness */
http_probe_done(hg, hp, reason);
return;
}
/* So: we did not connect and there is an error reason */
log_assert(!connects && reason);
/* try the next address */
if(hp->addr && ldns_rr_list_rr_count(hp->addr) != 0) {
http_probe_start_http_get(hp);
return;
}
/* no more addresses? try the next url */
if(hp->url_idx+1 < global_svr->http->url_num) {
http_probe_go_next_url(hg, hp, NULL);
return;
}
/* fail */
http_probe_done(hg, hp, reason);
}
static int
http_probe_create_get(struct http_probe* hp, ldns_rr* addr, char** reason)
{
struct probe_ip* p;
p = (struct probe_ip*)calloc(1, sizeof(*p));
if(!p) {
*reason = "out of memory";
return 0;
}
p->port = hp->port;
p->to_http = 1;
p->http_ip6 = hp->ip6;
if(!addr || !ldns_rr_rdf(addr, 0)) {
*reason = "addr without rdata";
free(p);
return 0;
}
p->name = ldns_rdf2str(ldns_rr_rdf(addr, 0));
if(!p->name) {
free(p);
*reason = "out of memory";
return 0;
}
/* create http_get structure */
p->http = http_get_create(hp->url, global_svr->base, p);
if(!p->http) {
*reason = "out of memory";
free(p->name);
free(p);
return 0;
}
/* put a cap on the max data size because we expect very short
* responses for our probe */
p->http->data_limit = MAX_HTTP_LENGTH*10;
if(!http_get_fetch(p->http, p->name, hp->port, reason)) {
http_get_delete(p->http);
free(p->name);
free(p);
return 0;
}
p->http_desc = strdup(hp->hostname);
if(!p->http_desc) {
*reason = "malloc failure";
http_get_delete(p->http);
free(p->name);
free(p);
return 0;
}
/* put it in the svr list */
p->next = global_svr->probes;
global_svr->probes = p;
global_svr->num_probes++;
return 1;
}
/** pick random address from ldns_rr_list and remove it from the list */
ldns_rr* http_pick_random_addr(ldns_rr_list* list)
{
size_t count;
size_t i;
ldns_rr* rr;
if(!list) return NULL;
count = ldns_rr_list_rr_count(list);
if(count == 0) return NULL;
i = (count == 1)?0:(ldns_get_random()%count);
rr = ldns_rr_list_rr(list, i);
/* remove from rr_list */
(void)ldns_rr_list_set_rr(list, ldns_rr_list_rr(list, count-1), i);
ldns_rr_list_set_rr_count(list, count-1);
return rr;
}
/** start http get with a random dest address from the set */
void http_probe_start_http_get(struct http_probe* hp)
{
char* reason = "out of memory";
/* pick random address */
ldns_rr* rr = http_pick_random_addr(hp->addr);
/* create probe */
if(!http_probe_create_get(hp, rr, &reason)) {
log_err("http_probe_create_get: %s", reason);
ldns_rr_free(rr);
http_probe_done_addr(global_svr->http, hp, reason, 0, NULL);
return;
}
ldns_rr_free(rr);
}
/** delete http probe structure */
static void http_probe_delete(struct http_probe* hp)
{
if(!hp) return;
ldns_rr_list_deep_free(hp->addr);
free(hp->url);
free(hp->hostname);
free(hp->filename);
free(hp);
}
/** create and start new http probe for v4 or v6 */
static struct http_probe*
http_probe_start(struct http_general* hg, int ip6)
{
struct http_probe* hp = (struct http_probe*)calloc(1, sizeof(*hp));
if(!hp) return NULL;
hp->ip6 = ip6;
hp->port = HTTP_PORT;
if(!http_probe_setup_url(hg, hp, 0)) {
http_probe_delete(hp);
return NULL;
}
if(http_probe_hostname_has_addr(hp)) {
if(!hp->addr || ldns_rr_list_rr_count(hp->addr)==0) {
if(hp->ip6)
http_probe_done(hg, hp, "no address of type IP6");
else http_probe_done(hg, hp, "no address of type IP4");
return hp;
}
hp->got_addrs = 1;
hp->do_addr = 0;
http_probe_start_http_get(hp);
return hp;
}
http_probe_make_addr_queries(hg, hp);
return hp;
}
/* see if str in array */
static int already_used(struct http_general* hg, char* s)
{
size_t i;
for(i=0; i<hg->url_num; i++)
if(hg->urls[i] == s)
return 1;
return 0;
}
/* pick url from list not picking twice */
static char* pick_url(struct http_general* hg, size_t x, char** code)
{
size_t now = 0;
struct strlist2* p;
for(p=hg->svr->cfg->http_urls; p; p=p->next) {
if(already_used(hg, p->str1))
continue;
if(now++ == x) {
*code = p->str2;
return p->str1;
}
}
return NULL;
}
/* fill the url array randomly */
static void fill_urls(struct http_general* hg)
{
size_t i;
for(i=0; i<hg->url_num; i++) {
/* random number from remaining number of choices */
if((int)hg->url_num == hg->svr->cfg->num_http_urls &&
i == hg->url_num)
hg->urls[i] = pick_url(hg, 0, &hg->codes[i]);
else hg->urls[i] = pick_url(hg,
ldns_get_random()%(hg->svr->cfg->num_http_urls - i),
&hg->codes[i]);
}
for(i=0; i<hg->url_num; i++)
verbose(VERB_ALGO, "hg url[%d]=%s", (int)i, hg->urls[i]);
}
struct http_general* http_general_start(struct svr* svr)
{
struct http_general* hg = (struct http_general*)calloc(1, sizeof(*hg));
if(!hg) return NULL;
hg->svr = svr;
if(svr->cfg->num_http_urls >= HTTP_NUM_URLS_MAX_PROBE)
hg->url_num = HTTP_NUM_URLS_MAX_PROBE;
else hg->url_num = (size_t)svr->cfg->num_http_urls;
hg->urls = (char**)calloc(hg->url_num, sizeof(char*));
hg->codes = (char**)calloc(hg->url_num, sizeof(char*));
if(!hg->urls || !hg->codes) {
free(hg->urls);
free(hg->codes);
free(hg);
return NULL;
}
/* randomly pick that number of urls from the config */
fill_urls(hg);
/* start v4 and v6 */
hg->v4 = http_probe_start(hg, 0);
if(!hg->v4) {
log_err("out of memory");
http_general_delete(hg);
return NULL;
}
hg->v6 = http_probe_start(hg, 1);
if(!hg->v6) {
log_err("out of memory");
http_general_delete(hg);
return NULL;
}
return hg;
}
void http_general_delete(struct http_general* hg)
{
if(!hg) return;
free(hg->urls);
free(hg->codes);
http_probe_delete(hg->v4);
http_probe_delete(hg->v6);
free(hg);
}
void http_general_done(const char* reason)
{
struct svr* svr = global_svr;
verbose(VERB_OPS, "http_general done %s", reason?reason:"success");
if(!reason) {
svr->http->saw_http_work = 1;
}
if(svr->num_probes_done < svr->num_probes) {
/* if we are probing the cache, and now http works,
* and some cache was already seen to work.
* (and we are not probing TCP, SSL, Authority),
* (and we are not in forced_insecure mode).
* Then we can already use the working cache server now. */
if(!reason && !svr->probe_dnstcp && !svr->probe_direct &&
svr->saw_first_working && !svr->forced_insecure) {
probe_setup_cache(svr, NULL);
}
return; /* wait for other probes at the cache stage */
}
probe_cache_done();
}
void http_host_outq_done(struct probe_ip* p, const char* reason)
{
struct http_probe* hp;
if(!reason) {
verbose(VERB_OPS, "addr lookup %s at %s successful",
p->host_c->qname, p->name);
p->works = 1;
} else {
verbose(VERB_OPS, "addr lookup %s at %s failed: %s",
p->host_c->qname, p->name, reason);
p->reason = strdup(reason);
p->works = 0;
}
p->finished = 1;
global_svr->num_probes_done++;
if(p->http_ip6)
hp = global_svr->http->v6;
else hp = global_svr->http->v4;
if(reason) {
hp->num_failed_addr_qs++;
/* see if other address lookups have also failed */
if(hp->num_failed_addr_qs >= hp->num_addr_qs) {
/* if so, go to next url */
/* attempt to go to the next url or fail if no next url */
if(hp->url_idx+1 < global_svr->http->url_num) {
http_probe_remove_addr_lookups(hp);
http_probe_go_next_url(global_svr->http, hp,
NULL);
} else {
http_probe_done(global_svr->http, hp,
"cannot resolve domain name");
}
}
} else {
hp->got_addrs = 1;
hp->do_addr = 0;
/* it worked, remove other address lookups for this hostname */
http_probe_remove_addr_lookups(hp);
/* if it worked, then start http part of the probe sequence */
http_probe_start_http_get(hp);
}
}
void http_host_outq_result(struct probe_ip* p, ldns_pkt* pkt)
{
/* not picked by name because of CNAMEs */
ldns_rr_list* addr = ldns_pkt_rr_list_by_type(pkt, p->http_ip6?
LDNS_RR_TYPE_AAAA:LDNS_RR_TYPE_A, LDNS_SECTION_ANSWER);
ldns_pkt_free(pkt);
/* see if RR data, is empty, outq_done with error */
if(!addr || ldns_rr_list_rr_count(addr) == 0) {
ldns_rr_list_deep_free(addr);
http_host_outq_done(p, "nodata answer");
return;
}
/* store the address results */
if(p->http_ip6)
global_svr->http->v6->addr = addr;
else global_svr->http->v4->addr = addr;
http_host_outq_done(p, NULL);
}
/** check if the data is correct, ignore whitespace */
static int
hg_check_data(ldns_buffer* data, char* result)
{
char* s = (char*)ldns_buffer_begin(data);
size_t result_len = strlen(result);
while(isspace(*s))
s++;
if(strncmp(s, result, result_len) != 0)
return 0;
/*
* check that there is nothing else
* than whitespaces after the expected string
*/
s += result_len;
while(isspace(*s))
s++;
if(*s != 0)
return 0;
return 1;
}
/** http get is done (failure or success) */
static void
http_get_done(struct http_get* hg, char* reason, int connects, char* redirect)
{
struct probe_ip* p;
struct http_probe* hp;
char* redirect_dup = NULL;
if(!hg->probe) {
/* update http_get */
if(redirect && !reason)
reason = "error redirect for selfupdate";
selfupdate_http_get_done(global_svr->update, hg, reason);
return;
}
p = hg->probe;
hp = (p->http_ip6)?global_svr->http->v6:global_svr->http->v4;
p->finished = 1;
global_svr->num_probes_done++;
/* printout data we got (but pages can be big)
if(!reason) verbose(VERB_ALGO, "got %d data: '%s'",
(int)ldns_buffer_position(hg->data),
ldns_buffer_begin(hg->data)); */
if(!reason || connects)
hp->connects = 1;
if(!reason && !redirect) {
/* check the data */
if(!hg_check_data(hg->data,
global_svr->http->codes[hp->url_idx]))
reason = "wrong page content";
else verbose(VERB_ALGO, "correct page content from %s",
p->name);
}
if(redirect) {
redirect_dup = strdup(redirect);
if(!redirect_dup) reason = "out of memory";
}
verbose(VERB_OPS, "http_get_done: %s from %s: %s (%s%s%s)", hg->url, hg->dest,
reason?reason:"success",
!reason||connects?"connects":"noconnects",
redirect?" redirected to ":"",
redirect?redirect:"");
if(!reason && !redirect) {
p->works = 1;
} else if(redirect_dup) {
/* do not set p->works otherwise, dupped already */
char buf[1024];
snprintf(buf, sizeof(buf), "http redirect to %s", redirect);
p->reason = strdup(buf);
if(!p->reason) log_err("malloc failure");
} else {
p->works = 0;
p->reason = strdup(reason);
if(!p->reason) log_err("malloc failure");
}
http_get_delete(hg);
p->http = NULL;
http_probe_done_addr(global_svr->http, hp, reason, hp->connects,
redirect_dup);
}
/** handle timeout for the http_get operation */
void
http_get_timeout_handler(void* arg)
{
struct http_get* hg = (struct http_get*)arg;
verbose(VERB_ALGO, "http_get timeout");
http_get_done(hg, "timeout", 0, NULL);
}
struct http_get* http_get_create(const char* url, struct comm_base* base,
struct probe_ip* probe)
{
struct http_get* hg = (struct http_get*)calloc(1, sizeof(*hg));
if(!hg) {
log_err("http_get_create: out of memory");
return NULL;
}
hg->probe = probe;
hg->state = http_state_none;
hg->url = strdup(url);
if(!hg->url) {
log_err("http_get_create: out of memory");
free(hg);
return NULL;
}
hg->buf = ldns_buffer_new(MAX_HTTP_LENGTH);
if(!hg->buf) {
log_err("http_get_create: out of memory");
http_get_delete(hg);
return NULL;
}
hg->data = ldns_buffer_new(MAX_HTTP_LENGTH);
if(!hg->data) {
log_err("http_get_create: out of memory");
http_get_delete(hg);
return NULL;
}
hg->timer = comm_timer_create(base, http_get_timeout_handler, hg);
if(!hg->timer) {
log_err("http_get_create: out of memory");
http_get_delete(hg);
return NULL;
}
hg->base = base;
return hg;
}
/** Put HTTP GET 1.1 into the buffer, ready to send to host */
static int
prep_get_cmd(struct http_get* hg)
{
ldns_buffer_clear(hg->buf);
if(ldns_buffer_printf(hg->buf, "GET /%s HTTP/1.1\r\n", hg->filename)
== -1)
return 0;
if(ldns_buffer_printf(hg->buf, "Host: %s\r\n", hg->hostname) == -1)
return 0;
if(ldns_buffer_printf(hg->buf, "User-Agent: dnssec-trigger/%s\r\n",
PACKAGE_VERSION) == -1)
return 0;
/* not really needed: Connection: close */
if(ldns_buffer_printf(hg->buf, "\r\n") == -1)
return 0;
ldns_buffer_flip(hg->buf);
/* verbose(VERB_ALGO, "created http get text: %s", ldns_buffer_begin(hg->buf)); */
return 1;
}
/** connect to destination IP (nonblocking), return fd (or -1). */
static int
http_get_connect(struct sockaddr_storage* addr, socklen_t addrlen, char** err)
{
int fd;
#ifdef INET6
if(addr_is_ip6(addr, addrlen))
fd = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
else
#endif
fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if(fd == -1) {
#ifndef USE_WINSOCK
*err = strerror(errno);
if(!((errno == EAFNOSUPPORT || errno == EPROTONOSUPPORT) &&
verbosity <= 2))
log_err("http_get: socket: %s", strerror(errno));
#else
*err = wsa_strerror(WSAGetLastError());
if(!((WSAGetLastError() == WSAEAFNOSUPPORT ||
WSAGetLastError() == WSAEPROTONOSUPPORT) &&
verbosity <= 2))
log_err("http_get: socket: %s",
wsa_strerror(WSAGetLastError()));
#endif
return -1;
}
fd_set_nonblock(fd);
if(connect(fd, (struct sockaddr*)addr, addrlen) == -1) {
#ifndef USE_WINSOCK
#ifdef EINPROGRESS
if(errno != EINPROGRESS) {
#else
if(1) {
#endif
*err = strerror(errno);
if(verbosity >= 2)
log_err("http_get: connect: %s", *err);
close(fd);
#else /* USE_WINSOCK */
if(WSAGetLastError() != WSAEINPROGRESS &&
WSAGetLastError() != WSAEWOULDBLOCK) {
*err = wsa_strerror(WSAGetLastError());
if(verbosity >= 2)
log_err("http_get: connect: %s", *err);
closesocket(fd);
#endif
return -1;
}
}
return fd;
}
/** write buffer to socket, returns true if done, false if notdone or error */
static int hg_write_buf(struct http_get* hg, ldns_buffer* buf)
{
ssize_t r;
char* str = NULL;
int fd = hg->cp->fd;
if(hg->cp->tcp_check_nb_connect) {
/* check for pending error from nonblocking connect */
/* from Stevens, unix network programming, vol1, 3rd ed, p450*/
int error = 0;
socklen_t len = (socklen_t)sizeof(error);
if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&error,\
&len) < 0) {
#ifndef USE_WINSOCK
error = errno; /* on solaris errno is error */
#else /* USE_WINSOCK */
error = WSAGetLastError();
#endif
}
#ifndef USE_WINSOCK
#if defined(EINPROGRESS) && defined(EWOULDBLOCK)
if(error == EINPROGRESS || error == EWOULDBLOCK)
return 0; /* try again later */
else
#endif
if(error != 0) {
str = strerror(error);
#else /* USE_WINSOCK */
if(error == WSAEINPROGRESS)
return 0;
else if(error == WSAEWOULDBLOCK) {
winsock_tcp_wouldblock(comm_point_internal(hg->cp),
EV_WRITE);
return 0;
} else if(error != 0) {
str = wsa_strerror(error);
#endif /* USE_WINSOCK */
log_err("http connect: %s", str);
http_get_done(hg, str, 0, NULL);
return 0;
}
/* no connect error */
hg->cp->tcp_check_nb_connect = 0;
if(!hg->probe) {
selfupdate_http_connected(global_svr->update, hg);
}
}
/* write data */
r = send(fd, (void*)ldns_buffer_current(buf),
ldns_buffer_remaining(buf), 0);
if(r == -1) {
#ifndef USE_WINSOCK
if(errno == EINTR || errno == EAGAIN)
return 0;
str = strerror(errno);
#else
if(WSAGetLastError() == WSAEINPROGRESS)
return 0;
if(WSAGetLastError() == WSAEWOULDBLOCK) {
winsock_tcp_wouldblock(comm_point_internal(hg->cp),
EV_WRITE);
return 0;
}
str = wsa_strerror(WSAGetLastError());
#endif
log_err("http write: %s", str);
http_get_done(hg, str, 0, NULL);
return 0;
}
ldns_buffer_skip(buf, r);
return (ldns_buffer_remaining(buf) == 0);
}
/** read buffer from socket, returns false on failures (or simply not done).
* returns true if something extra was read in. caller checks if done.
* zero terminates after a read (right after buf position) */
static int hg_read_buf(struct http_get* hg, ldns_buffer* buf)
{
ssize_t r;
int fd = hg->cp->fd;
/* save up one space at end for a trailing zero byte */
r = recv(fd, (void*)ldns_buffer_current(buf),
ldns_buffer_remaining(buf)-1, 0);
/* zero terminate for sure */
ldns_buffer_write_u8_at(buf, ldns_buffer_limit(buf)-1, 0);
/* check for errors */
if(r == -1) {
char* str = NULL;
#ifndef USE_WINSOCK
if(errno == EINTR || errno == EAGAIN)
return 0;
str = strerror(errno);
#else
if(WSAGetLastError() == WSAEINPROGRESS)
return 0;
if(WSAGetLastError() == WSAEWOULDBLOCK) {
winsock_tcp_wouldblock(comm_point_internal(hg->cp),
EV_READ);
return 0;
}
/* WSAECONNRESET could happen */
str = wsa_strerror(WSAGetLastError());
#endif
log_err("http read: %s", str);
http_get_done(hg, str, 0, NULL);
return 0;
}
if(r == 0) {
log_err("http read: stream closed");
http_get_done(hg, "stream closed", 0, NULL);
return 0;
}
ldns_buffer_skip(buf, r);
return 1;
}
/** parse lines from the buffer, from begin to len, modify to remove line-end
* and call parse func. Returns false on parse failure. */
static int
hg_parse_lines(struct http_get* hg, ldns_buffer* src, size_t len,
int (*func)(struct http_get*, char*, void*), void* arg)
{
size_t pos = 0;
log_assert(len <= ldns_buffer_position(src));
/* the check for pos in limit is for robustness */
while(pos < len && pos <= ldns_buffer_limit(src)) {
/* see if there is a (whole) line here) */
/* safe because the source buffer is zero terminated for sure*/
char* eol = strstr((char*)ldns_buffer_at(src, pos), "\r\n");
if(!eol) {
http_get_done(hg, "header line without eol", 1, NULL);
return 0;
}
if(pos >= ldns_buffer_limit(src)) {
/* impossible, but check for robustness */
http_get_done(hg, "header line too long", 1, NULL);
return 0;
}
if(eol > (char*)ldns_buffer_at(src, len)) {
break;
}
/* eol == len is allowed, for \r\n at end of headers */
/* zero terminate the line (removes the eol from string) */
*eol = 0;
/* callback function */
if(!(*func)(hg, (char*)ldns_buffer_at(src, pos), arg))
return 0;
/* next line */
pos += (eol - (char*)ldns_buffer_at(src, pos));
pos += 2; /* skip -r-n (now 00-n) */
}
return 1;
}
/** move trailing end of buffer to the front */
static void
hg_buf_move(ldns_buffer* buf, size_t headlen)
{
if(ldns_buffer_position(buf) > headlen) {
size_t traillen = ldns_buffer_position(buf)-headlen;
memmove(ldns_buffer_begin(buf),
ldns_buffer_at(buf, headlen), traillen);
ldns_buffer_set_position(buf, traillen);
} else {
ldns_buffer_clear(buf);
}
/* zero terminate buffer */
ldns_buffer_current(buf)[0] = 0;
}
/** handle write of http request */
static int hg_handle_request(struct http_get* hg)
{
/* write request */
if(!hg_write_buf(hg, hg->buf))
return 0;
/* done, start reading reply headers */
ldns_buffer_clear(hg->buf);
comm_point_listen_for_rw(hg->cp, 1, 0);
hg->state = http_state_reply_header;
return 1;
}
/** parse reply header line */
static int
reply_header_parse(struct http_get* hg, char* line, void* arg)
{
size_t* datalen = (size_t*)arg;
verbose(VERB_ALGO, "http reply header: %s", line);
if(strncasecmp(line, "HTTP/1.1 ", 9) == 0) {
/* check returncode; we understand the following from
* rcodes:
* 2xx : success, look at content (perhaps changed by hotspot)
* 3xx : redirect of some form - probably the hotspot.
* other: failure
*/
if(strncmp(line+9, "302", 3)==0 ||
strncmp(line+9, "301", 3)==0 ||
strncmp(line+9, "303", 3)==0 ||
strncmp(line+9, "305", 3)==0 ||
strncmp(line+9, "307", 3)==0) {
/* redirect 302 (temporary), 303 (see other),
* 307(Temporariy Redirect). */
/* 305(proxy) we treat the same */
/* 301 moved permanently should not really be used
* by a hot spot .. */
hg->redirect_now = 1;
} else if(line[9] == '3') {
/* other redirect type codes, this means it fails
* completely*/
char err[512];
snprintf(err, sizeof(err), "http redirect %s", line+9);
/* we connected to the server, this looks like a
* hotspot that redirects */
http_get_done(hg, err, 1, NULL);
return 0;
} else if(line[9] != '2') {
char err[512];
snprintf(err, sizeof(err), "http error %s", line+9);
/* we 'connected' but it seems the page is not
* there anymore. Try another url and pretend we
* could not connect to get it to try another url. */
/* because we pass noconnect, it will also try other
* ip addresses for the server. perhaps another server
* does not give 404? */
http_get_done(hg, err, 0, NULL);
return 0;
}
} else if(strncasecmp(line, "Content-Length: ", 16) == 0) {
*datalen = (size_t)atoi(line+16);
} else if(strncasecmp(line, "Transfer-Encoding: chunked", 19+7) == 0) {
*datalen = 0;
} else if(strncasecmp(line, "Location: ", 10) == 0
&& hg->redirect_now) {
/* skip whitespace before url */
char* url = line+10;
while(isspace(*url))
url++;
hg->redirect_now = 0;
http_get_done(hg, NULL, 1, line+10);
return 0;
}
return 1;
}
/** handle read of reply headers (the topmost headers) */
static int hg_handle_reply_header(struct http_get* hg)
{
size_t headlen = 0;
size_t datalen = 0;
char* endstr;
if(!hg_read_buf(hg, hg->buf))
return 0;
/* check if done */
endstr = strstr((char*)ldns_buffer_begin(hg->buf), "\r\n\r\n");
if(!endstr) {
/* at end (with zero marker) */
if(ldns_buffer_remaining(hg->buf)-1 == 0) {
http_get_done(hg, "http headers too large", 1, NULL);
return 0;
}
return 0;
}
headlen = (size_t)(endstr-(char*)ldns_buffer_begin(hg->buf));
verbose(VERB_ALGO, "http done, parse reply header");
/* done reading, parse it */
log_assert(strncmp(ldns_buffer_at(hg->buf, headlen), "\r\n\r\n", 4)==0);
/* there is a header part, and a start of a trailing part. */
/* extract lines, parse with the given function */
if(!hg_parse_lines(hg, hg->buf, headlen, reply_header_parse, &datalen))
return 0;
/* figured out what form the reply takes (one data and its length,
* or chunked, or error */
/* move trailing part to front of data buffer (skip /r/n/r/n) */
hg_buf_move(hg->buf, headlen+4);
/* if one data seg: see if data can fit into the buffer, or fail */
if(datalen != 0) {
if(hg->data_limit && datalen > hg->data_limit) {
http_get_done(hg, "http reply data too large", 1, NULL);
return 0;
}
if(!ldns_buffer_reserve(hg->buf,
datalen - ldns_buffer_position(hg->buf)+1)) {
http_get_done(hg, "out of memory", 1, NULL);
return 0;
}
hg->state = http_state_reply_data;
hg->datalen = datalen;
verbose(VERB_ALGO, "http 1.0 data len %d", (int)datalen);
} else {
hg->state = http_state_chunk_header;
}
return 1;
}
/** add data to output buffer */
static int
hg_add_data(struct http_get* hg, ldns_buffer* add, size_t len)
{
if(hg->data_limit && ldns_buffer_position(hg->data) + len >
hg->data_limit) {
http_get_done(hg, "http data too large", 1, NULL);
return 0;
}
if(!ldns_buffer_reserve(hg->data, len+1)) {
http_get_done(hg, "out of memory", 1, NULL);
return 0;
}
ldns_buffer_write(hg->data, ldns_buffer_begin(add), len);
/* zero terminate */
ldns_buffer_write_u8_at(hg->data, ldns_buffer_position(hg->data), 0);
return 1;
}
/** handle read of reply data (as one block of data) */
static int hg_handle_reply_data(struct http_get* hg)
{
/* this state could start with initial data that is complete
* already, otherwise, read more */
if(ldns_buffer_position(hg->buf) < hg->datalen) {
if(!hg_read_buf(hg, hg->buf))
return 0;
if(ldns_buffer_position(hg->buf) < hg->datalen)
return 0;
}
log_assert(ldns_buffer_position(hg->buf) >= hg->datalen);
if(!hg_add_data(hg, hg->buf, hg->datalen))
return 0;
/* done with success with data */
verbose(VERB_ALGO, "http read completed");
http_get_done(hg, NULL, 1, NULL);
return 0;
}
/** parse reply header line */
static int
chunk_header_parse(struct http_get* hg, char* line, void* arg)
{
size_t* chunklen = (size_t*)arg;
char* e = NULL;
size_t v;
verbose(VERB_ALGO, "http chunk header: '%s'", line);
v = (size_t)strtol(line, &e, 16);
if(e == line) {
http_get_done(hg, "could not parse chunk header", 1, NULL);
return 0;
}
*chunklen = v;
return 1;
}
/** handle read of chunked reply headers (the size of the chunk) */
static int hg_handle_chunk_header(struct http_get* hg)
{
/* this state could start with initial data that is complete
* already, otherwise, read more */
size_t headlen = 0;
size_t chunklen = 0;
char* endstr;
if(!strstr((char*)ldns_buffer_begin(hg->buf), "\r\n")) {
if(!hg_read_buf(hg, hg->buf))
return 0;
}
endstr = strstr((char*)ldns_buffer_begin(hg->buf), "\r\n");
if(!endstr) {
/* at end (with zero marker) */
if(ldns_buffer_remaining(hg->buf)-1 == 0) {
http_get_done(hg, "http chunk headers too large", 1, NULL);
return 0;
}
return 0;
}
headlen = (size_t)(endstr-(char*)ldns_buffer_begin(hg->buf));
/* done reading, parse it */
log_assert(strncmp(ldns_buffer_at(hg->buf, headlen), "\r\n", 2)==0);
/* extract lines, parse with the given function */
if(!hg_parse_lines(hg, hg->buf, headlen, chunk_header_parse, &chunklen))
return 0;
if(chunklen == 0) {
/* chunked read completed */
/* TODO there can be chunked trailer headers here .. */
verbose(VERB_ALGO, "http chunked read completed");
http_get_done(hg, NULL, 1, NULL);
return 0;
}
/* move trailing part to front of data buffer (skip /r/n) */
hg_buf_move(hg->buf, headlen+2);
/* see if data can possibly fit */
if(hg->data_limit && chunklen > hg->data_limit) {
http_get_done(hg, "http reply chunk data too large", 1, NULL);
return 0;
}
if(!ldns_buffer_reserve(hg->buf,
chunklen - ldns_buffer_position(hg->buf)+1)) {
http_get_done(hg, "out of memory", 1, NULL);
return 0;
}
hg->state = http_state_chunk_data;
verbose(VERB_ALGO, "http chunk len %d", (int)chunklen);
hg->datalen = chunklen;
return 1;
}
/** handle read of chunked reply data (of one chunk) */
static int hg_handle_chunk_data(struct http_get* hg)
{
/* this state could start with initial data that is complete
* already, otherwise, read more */
/* read datalen+2 - body + /r/n */
if(ldns_buffer_position(hg->buf) < hg->datalen+2) {
if(!hg_read_buf(hg, hg->buf))
return 0;
if(ldns_buffer_position(hg->buf) < hg->datalen+2)
return 0;
}
/* done reading put it together */
log_assert(ldns_buffer_position(hg->buf) >= hg->datalen);
verbose(VERB_ALGO, "datalen %d", (int)hg->datalen);
verbose(VERB_ALGO, "position %d", (int)ldns_buffer_position(hg->buf));
if(strncmp((char*)ldns_buffer_at(hg->buf, hg->datalen), "\r\n", 2)!=0) {
http_get_done(hg, "chunk data not terminated with eol", 1, NULL);
return 0;
}
/* remove trailing newline */
ldns_buffer_write_u8_at(hg->buf, hg->datalen, 0);
if(!hg_add_data(hg, hg->buf, hg->datalen))
return 0;
/* move up data (plus emptyline) */
hg_buf_move(hg->buf, hg->datalen+2);
hg->state = http_state_chunk_header;
return 1;
}
/** handle http get state (return true to continue processing) */
static int hg_handle_state(struct http_get* hg)
{
verbose(VERB_ALGO, "hg_handle_state %d", (int)hg->state);
switch(hg->state) {
case http_state_none:
/* not possible */
http_get_done(hg, "got event while not connected", 0, NULL);
return 0;
case http_state_request:
return hg_handle_request(hg);
case http_state_reply_header:
return hg_handle_reply_header(hg);
case http_state_reply_data:
return hg_handle_reply_data(hg);
case http_state_chunk_header:
return hg_handle_chunk_header(hg);
case http_state_chunk_data:
return hg_handle_chunk_data(hg);
default:
break;
}
http_get_done(hg, "unknown state", 0, NULL);
return 0;
}
/** handle events (read or write) on the http file descriptor */
int
http_get_callback(struct comm_point* ATTR_UNUSED(cp), void* arg, int err,
struct comm_reply* ATTR_UNUSED(reply))
{
struct http_get* hg = (struct http_get*)arg;
if(err != NETEVENT_NOERROR) {
/* timeout handled with timer, other errors not possible */
log_err("internal error: http_get_callback got %d", err);
return 0;
}
/* is this read or write, and if so, what part of the protocol */
verbose(VERB_ALGO, "http_get: got event for %s from %s", hg->url, hg->dest);
while(hg_handle_state(hg)) {
;
}
/* the return value is not used by comm_point_raw */
return 0;
}
int http_get_fetch(struct http_get* hg, const char* dest, int port, char** err)
{
int fd;
struct timeval tv;
struct sockaddr_storage addr;
socklen_t addrlen = 0;
/* parse the URL */
verbose(VERB_ALGO, "http_get fetch %s from %s", hg->url, dest);
if(!parse_url(hg->url, &hg->hostname, &hg->filename)) {
*err = "cannot parse url";
return 0;
}
verbose(VERB_ALGO, "parsed into %s and %s", hg->hostname, hg->filename);
/* parse dest IP address */
if(!(hg->dest = strdup(dest))) {
*err = "out of memory";
return 0;
}
hg->port = port;
if(!ipstrtoaddr(dest, port, &addr, &addrlen)) {
log_err("error in syntax of IP address %s", dest);
*err = "cannot parse IP address";
return 0;
}
/* start TCP connection to destination, prepare send headers */
if(!prep_get_cmd(hg)) {
*err = "out of memory";
return 0;
}
/* clear and zero terminate data buffer */
ldns_buffer_clear(hg->data);
ldns_buffer_write_u8_at(hg->data, ldns_buffer_position(hg->data), 0);
/* set timeout */
tv.tv_sec = HTTP_TIMEOUT/1000;
tv.tv_usec = HTTP_TIMEOUT%1000;
comm_timer_set(hg->timer, &tv);
/* create fd and connect nonblockingly */
if( (fd=http_get_connect(&addr, addrlen, err)) == -1) {
return 0;
}
/* commpoint, raw, get nb_connect check on write */
/* we created the fd before to pass it here, so that the event_add
* on windows sees the full (TCP, SOCK_STREAM) file descriptor and
* activates the necessary workarounds for TCP sticky events */
hg->cp = comm_point_create_raw(hg->base, fd, 1, http_get_callback, hg);
if(!hg->cp) {
*err = "out of memory";
return 0;
}
hg->cp->do_not_close = 0;
hg->cp->tcp_check_nb_connect = 1;
hg->state = http_state_request;
*err = NULL;
return 1;
}
void http_get_delete(struct http_get* hg)
{
if(!hg) return;
free(hg->url);
free(hg->hostname);
free(hg->filename);
free(hg->dest);
ldns_buffer_free(hg->buf);
ldns_buffer_free(hg->data);
comm_point_delete(hg->cp);
comm_timer_delete(hg->timer);
free(hg);
}