/*
* Copyright (c) 2008-2012 Zmanda, Inc. All Rights Reserved.
* Copyright (c) 2013-2016 Carbonite, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*
* Contact information: Carbonite Inc., 756 N Pastoria Ave
* Sunnyvale, CA 94085, or: http://www.zmanda.com
*/
/* TODO
* - collect speed statistics
* - debugging mode
*/
#ifdef HAVE_CONFIG_H
/* use a relative path here to avoid conflicting with Perl's config.h. */
#include "../config/config.h"
#endif
#include <string.h>
#include "s3.h"
#include "s3-util.h"
#ifdef HAVE_REGEX_H
#include <regex.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif
#ifdef HAVE_TIME_H
#include <time.h>
#endif
#ifdef HAVE_UTIL_H
#include "amutil.h"
#endif
#ifdef HAVE_AMANDA_H
#include "amanda.h"
#endif
#include "amjson.h"
#include <curl/curl.h>
/* Constant renamed after version 7.10.7 */
#ifndef CURLINFO_RESPONSE_CODE
#define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
#endif
/* We don't need OpenSSL's kerberos support, and it's broken in
* RHEL 3 anyway. */
#define OPENSSL_NO_KRB5
#ifdef HAVE_OPENSSL_HMAC_H
# include <openssl/hmac.h>
#else
# ifdef HAVE_CRYPTO_HMAC_H
# include <crypto/hmac.h>
# else
# ifdef HAVE_HMAC_H
# include <hmac.h>
# endif
# endif
#endif
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <openssl/md5.h>
char *S3_name[] = {
"UNKNOWN",
"S3",
"OpenStack",
"OpenStack",
"OpenStack",
"Google Storage",
"Castor",
"S3",
};
char *S3_bucket_name[] = {
"UNKNOWN",
"S3 bucket",
"OpenStack bucket",
"OpenStack bucket",
"OpenStack bucket",
"Google Storage bucket",
"Castor bucket",
"S3 bucket",
};
/* Maximum key length as specified in the S3 documentation
* (*excluding* null terminator) */
#define S3_MAX_KEY_LENGTH 1024
#define AMAZON_SECURITY_HEADER "x-amz-security-token"
#define AMAZON_STORAGE_CLASS_HEADER "x-amz-storage-class"
#define AMAZON_SERVER_SIDE_ENCRYPTION_HEADER "x-amz-server-side-encryption"
#define AMAZON_WILDCARD_LOCATION "*"
#define AMZ_QUOTA_CLOUD_URL ".s3.amazonaws.com"
/* parameters for exponential backoff in the face of retriable errors */
/* start at 0.01s */
#define EXPONENTIAL_BACKOFF_START_USEC G_USEC_PER_SEC/100
/* double at each retry */
#define EXPONENTIAL_BACKOFF_BASE 2
/* retry 14 times (for a total of about 3 minutes spent waiting) */
#define EXPONENTIAL_BACKOFF_MAX_RETRIES 14
/* general "reasonable size" parameters */
#define MAX_ERROR_RESPONSE_LEN (100*1024)
// CURLE_SSL_CACERT_BADFILE is defined in 7.16.0
#if LIBCURL_VERSION_NUM >= 0x071000
#define AMAMDA_CURLE_SSL_CACERT_BADFILE CURLE_SSL_CACERT_BADFILE
#else
# define AMAMDA_CURLE_SSL_CACERT_BADFILE CURLE_GOT_NOTHING
#endif
/* Results which should always be retried */
#define RESULT_HANDLING_ALWAYS_RETRY \
{ 400, S3_ERROR_RequestTimeout, 0, S3_RESULT_RETRY }, \
{ 403, S3_ERROR_RequestTimeTooSkewed,0, S3_RESULT_RETRY }, \
{ 409, S3_ERROR_OperationAborted, 0, S3_RESULT_RETRY }, \
{ 412, S3_ERROR_PreconditionFailed, 0, S3_RESULT_RETRY }, \
{ 429, 0, 0, S3_RESULT_RETRY }, \
{ 500, S3_ERROR_None, 0, S3_RESULT_RETRY }, \
{ 500, S3_ERROR_InternalError, 0, S3_RESULT_RETRY }, \
{ 501, S3_ERROR_NotImplemented, 0, S3_RESULT_RETRY }, \
{ 503, S3_ERROR_ServiceUnavailable, 0, S3_RESULT_RETRY }, \
{ 503, S3_ERROR_SlowDown, 0, S3_RESULT_RETRY }, \
{ 503, 0, 0, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_COULDNT_CONNECT, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_COULDNT_RESOLVE_HOST, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_PARTIAL_FILE, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_OPERATION_TIMEOUTED, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_SSL_CONNECT_ERROR, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_SEND_ERROR, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_RECV_ERROR, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_ABORTED_BY_CALLBACK, S3_RESULT_RETRY }, \
{ 0, 0, CURLE_GOT_NOTHING, S3_RESULT_RETRY }, \
{ 0, 0, AMAMDA_CURLE_SSL_CACERT_BADFILE, S3_RESULT_RETRY }
/*
* Data structures and associated functions
*/
struct S3Handle {
/* (all strings in this struct are freed by s3_free()) */
char *access_key;
char *secret_key;
char *session_token;
char *user_token;
char *swift_account_id;
char *swift_access_key;
char *username;
char *password;
char *tenant_id;
char *tenant_name;
char *project_name;
char *domain_name;
char *client_id;
char *client_secret;
char *refresh_token;
char *access_token;
time_t expires;
gboolean getting_oauth2_access_token;
gboolean getting_swift_2_token;
gboolean getting_swift_3_token;
/* attributes for new objects */
char *bucket_location;
char *storage_class;
char *server_side_encryption;
char *proxy;
char *host;
char *host_without_port;
char *service_path;
gboolean use_subdomain;
S3_api s3_api;
char *ca_info;
char *x_auth_token;
char *x_storage_url;
char *x_amz_expiration;
char *x_amz_restore;
CURL *curl;
gboolean verbose;
gboolean use_ssl;
gboolean server_side_encryption_header;
guint64 max_send_speed;
guint64 max_recv_speed;
/* information from the last request */
char *last_message;
guint last_response_code;
s3_error_code_t last_s3_error_code;
CURLcode last_curl_code;
guint last_num_retries;
void *last_response_body;
guint last_response_body_size;
char *uploadId;
char *etag;
/* offset with s3 */
time_t time_offset_with_s3;
char *content_type;
gboolean reuse_connection;
gboolean read_from_glacier;
char *transfer_encoding;
long timeout;
/* CAStor */
char *reps;
char *reps_bucket;
};
typedef struct {
CurlBuffer resp_buf;
s3_write_func write_func;
s3_reset_func reset_func;
gpointer write_data;
gboolean headers_done;
gboolean int_write_done;
char *etag;
/* Points to current handle: Added to get hold of s3 offset */
struct S3Handle *hdl;
} S3InternalData;
/* Callback function to examine headers one-at-a-time
*
* @note this is the same as CURLOPT_HEADERFUNCTION
*
* @param data: The pointer to read data from
* @param size: The size of each "element" of the data buffer in bytes
* @param nmemb: The number of elements in the data buffer.
* So, the buffer's size is size*nmemb bytes.
* @param stream: the header_data (an opaque pointer)
*
* @return The number of bytes written to the buffer or
* CURL_WRITEFUNC_PAUSE to pause.
* If it's the number of bytes written, it should match the buffer size
*/
typedef size_t (*s3_header_func)(void *data, size_t size, size_t nmemb, void *stream);
/*
* S3 errors */
/* (see preprocessor magic in s3.h) */
static char * s3_error_code_names[] = {
#define S3_ERROR(NAME) #NAME
S3_ERROR_LIST
#undef S3_ERROR
};
/* Convert an s3 error name to an error code. This function
* matches strings case-insensitively, and is appropriate for use
* on data from the network.
*
* @param s3_error_code: the error name
* @returns: the error code (see constants in s3.h)
*/
static s3_error_code_t
s3_error_code_from_name(char *s3_error_name);
/* Convert an s3 error code to a string
*
* @param s3_error_code: the error code to convert
* @returns: statically allocated string
*/
static const char *
s3_error_name_from_code(s3_error_code_t s3_error_code);
static void s3_new_curl(S3Handle *hdl);
/*
* result handling */
/* result handling is specified by a static array of result_handling structs,
* which match based on response_code (from HTTP) and S3 error code. The result
* given for the first match is used. 0 acts as a wildcard for both response_code
* and s3_error_code. The list is terminated with a struct containing 0 for both
* response_code and s3_error_code; the result for that struct is the default
* result.
*
* See RESULT_HANDLING_ALWAYS_RETRY for an example.
*/
typedef enum {
S3_RESULT_RETRY = -1,
S3_RESULT_FAIL = 0,
S3_RESULT_OK = 1,
S3_RESULT_NOTIMPL = 2
} s3_result_t;
typedef struct result_handling {
guint response_code;
s3_error_code_t s3_error_code;
CURLcode curl_code;
s3_result_t result;
} result_handling_t;
/*
* get the access token for OAUTH2
*/
static gboolean oauth2_get_access_token(S3Handle *hdl);
/* Lookup a result in C{result_handling}.
*
* @param result_handling: array of handling specifications
* @param response_code: response code from operation
* @param s3_error_code: s3 error code from operation, if any
* @param curl_code: the CURL error, if any
* @returns: the matching result
*/
static s3_result_t
lookup_result(const result_handling_t *result_handling,
guint response_code,
s3_error_code_t s3_error_code,
CURLcode curl_code);
/*
* Precompiled regular expressions */
static regex_t etag_regex, error_name_regex, message_regex, subdomain_regex,
location_con_regex, date_sync_regex, x_auth_token_regex,
x_subject_token_regex,
x_storage_url_regex, access_token_regex, expires_in_regex,
content_type_regex, details_regex, code_regex, uploadId_regex,
json_message_regex, html_error_name_regex, html_message_regex,
transfer_encoding_regex, x_amz_expiration_regex, x_amz_restore_regex;
/*
* Utility functions
*/
/* Check if a string is non-empty
*
* @param str: string to check
* @returns: true iff str is non-NULL and not "\0"
*/
static gboolean is_non_empty_string(const char *str);
char *am_strrmspace(char *str);
/* Construct the URL for an Amazon S3 REST request.
*
* A new string is allocated and returned; it is the responsiblity of the caller.
*
* @param hdl: the S3Handle object
* @param service_path: A path to add in the URL, or NULL for none.
* @param bucket: the bucket being accessed, or NULL for none
* @param key: the key being accessed, or NULL for none
* @param subresource: the sub-resource being accessed (e.g. "acl"), or NULL for none
* @param query: the query being accessed (e.g. "acl"), or NULL for none
*
* !use_subdomain: http://host/service_path/bucket/key
* use_subdomain : http://bucket.host/service_path/key
*
*/
static char *
build_url(
S3Handle *hdl,
const char *bucket,
const char *key,
const char *subresource,
const char **query);
/* Create proper authorization headers for an Amazon S3 REST
* request to C{headers}.
*
* @note: C{X-Amz} headers (in C{headers}) must
* - be in lower-case
* - be in alphabetical order
* - have no spaces around the colon
* (don't yell at me -- see the Amazon Developer Guide)
*
* @param hdl: the S3Handle object
* @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
* @param bucket: the bucket being accessed, or NULL for none
* @param key: the key being accessed, or NULL for none
* @param subresource: the sub-resource being accessed (e.g. "acl"), or NULL for none
* @param md5_hash: the MD5 hash of the request body, or NULL for none
*/
static struct curl_slist *
authenticate_request(S3Handle *hdl,
const char *verb,
const char *bucket,
const char *key,
const char *subresource,
const char **query,
const char *md5_hash,
const char *data_SHA256Hash,
const char *content_type,
const size_t content_length,
const char *project_id);
/* Interpret the response to an S3 operation, assuming CURL completed its request
* successfully. This function fills in the relevant C{hdl->last*} members.
*
* @param hdl: The S3Handle object
* @param body: the response body
* @param body_len: the length of the response body
* @param etag: The response's ETag header
* @param content_md5: The hex-encoded MD5 hash of the request body,
* which will be checked against the response's ETag header.
* If NULL, the header is not checked.
* If non-NULL, then the body should have the response headers at its beginnning.
* @returns: TRUE if the response should be retried (e.g., network error)
*/
static gboolean
interpret_response(S3Handle *hdl,
CURLcode curl_code,
char *curl_error_buffer,
gchar *body,
guint body_len,
const char *etag,
const char *content_md5);
/* Perform an S3 operation. This function handles all of the details
* of retryig requests and so on.
*
* The concepts of bucket and keys are defined by the Amazon S3 API.
* See: "Components of Amazon S3" - API Version 2006-03-01 pg. 8
*
* Individual sub-resources are defined in several places. In the REST API,
* they they are represented by a "flag" in the "query string".
* See: "Constructing the CanonicalizedResource Element" - API Version 2006-03-01 pg. 60
*
* @param hdl: the S3Handle object
* @param verb: the HTTP request method
* @param bucket: the bucket to access, or NULL for none
* @param key: the key to access, or NULL for none
* @param subresource: the "sub-resource" to request (e.g. "acl") or NULL for none
* @param query: NULL terminated array of query string, the must be in alphabetid order
* or NULL for none
* @param read_func: the callback for reading data
* Will use s3_empty_read_func if NULL is passed in.
* @param read_reset_func: the callback for to reset reading data
* @param size_func: the callback to get the number of bytes to upload
* @param md5_func: the callback to get the MD5 hash of the data to upload
* @param read_data: pointer to pass to the above functions
* @param write_func: the callback for writing data.
* Will use s3_counter_write_func if NULL is passed in.
* @param write_reset_func: the callback for to reset writing data
* @param write_data: pointer to pass to C{write_func}
* @param progress_func: the callback for progress information
* @param progress_data: pointer to pass to C{progress_func}
* @param result_handling: instructions for handling the results; see above.
* @returns: the result specified by result_handling; details of the response
* are then available in C{hdl->last*}
*/
static s3_result_t
perform_request(S3Handle *hdl,
const char *verb,
const char *bucket,
const char *key,
const char *subresource,
const char **query,
const char *content_type,
const char *project_id,
struct curl_slist *user_headers,
s3_read_func read_func,
s3_reset_func read_reset_func,
s3_size_func size_func,
s3_md5_func md5_func,
gpointer read_data,
s3_write_func write_func,
s3_reset_func write_reset_func,
gpointer write_data,
s3_progress_func progress_func,
gpointer progress_data,
const result_handling_t *result_handling,
gboolean chunked);
/*
* a CURLOPT_WRITEFUNCTION to save part of the response in memory and
* call an external function if one was provided.
*/
static size_t
s3_internal_write_func(void *ptr, size_t size, size_t nmemb, void * stream);
/*
* a function to reset to our internal buffer
*/
static void
s3_internal_reset_func(void * stream);
/*
* a CURLOPT_HEADERFUNCTION to save the ETag header only.
*/
static size_t
s3_internal_header_func(void *ptr, size_t size, size_t nmemb, void * stream);
static gboolean
compile_regexes(void);
static gboolean get_openstack_swift_api_v1_setting(S3Handle *hdl);
static gboolean get_openstack_swift_api_v2_setting(S3Handle *hdl);
static gboolean get_openstack_swift_api_v3_setting(S3Handle *hdl);
/*
* Static function implementations
*/
static s3_error_code_t
s3_error_code_from_name(char *s3_error_name)
{
int i;
if (!s3_error_name) return S3_ERROR_Unknown;
/* do a brute-force search through the list, since it's not sorted */
for (i = 0; i < S3_ERROR_END; i++) {
if (g_ascii_strcasecmp(s3_error_name, s3_error_code_names[i]) == 0)
return i;
}
return S3_ERROR_Unknown;
}
static const char *
s3_error_name_from_code(s3_error_code_t s3_error_code)
{
if (s3_error_code >= S3_ERROR_END)
s3_error_code = S3_ERROR_Unknown;
return s3_error_code_names[s3_error_code];
}
gboolean
s3_curl_supports_ssl(void)
{
static int supported = -1;
if (supported == -1) {
#if defined(CURL_VERSION_SSL)
curl_version_info_data *info = curl_version_info(CURLVERSION_NOW);
if (info->features & CURL_VERSION_SSL)
supported = 1;
else
supported = 0;
#else
supported = 0;
#endif
}
return supported;
}
static gboolean
s3_curl_throttling_compat(void)
{
/* CURLOPT_MAX_SEND_SPEED_LARGE added in 7.15.5 */
#if LIBCURL_VERSION_NUM >= 0x070f05
curl_version_info_data *info;
/* check the runtime version too */
info = curl_version_info(CURLVERSION_NOW);
return info->version_num >= 0x070f05;
#else
return FALSE;
#endif
}
static s3_result_t
lookup_result(const result_handling_t *result_handling,
guint response_code,
s3_error_code_t s3_error_code,
CURLcode curl_code)
{
while (result_handling->response_code
|| result_handling->s3_error_code
|| result_handling->curl_code) {
if ((result_handling->response_code && result_handling->response_code != response_code)
|| (result_handling->s3_error_code && result_handling->s3_error_code != s3_error_code)
|| (result_handling->curl_code && result_handling->curl_code != curl_code)) {
result_handling++;
continue;
}
return result_handling->result;
}
/* return the result for the terminator, as the default */
return result_handling->result;
}
static time_t
rfc3339_date(
const char *date)
{
gint year, month, day, hour, minute, seconds;
const char *atz;
if (strlen(date) < 19)
return 1073741824;
year = atoi(date);
month = atoi(date+5);
day = atoi(date+8);
hour = atoi(date+11);
minute = atoi(date+14);
seconds = atoi(date+17);
atz = date+19;
if (*atz == '.') { /* skip decimal seconds */
atz++;
while (*atz >= '0' && *atz <= '9') {
atz++;
}
}
#if GLIB_CHECK_VERSION(2,26,0)
if (!glib_check_version(2,26,0)) {
GTimeZone *tz;
GDateTime *dt;
time_t a;
tz = g_time_zone_new(atz);
dt = g_date_time_new(tz, year, month, day, hour, minute, seconds);
a = g_date_time_to_unix(dt);
g_time_zone_unref(tz);
g_date_time_unref(dt);
return a;
} else
#endif
{
struct tm tm;
time_t t;
tm.tm_year = year - 1900;
tm.tm_mon = month - 1;
tm.tm_mday = day;
tm.tm_hour = hour;
tm.tm_min = minute;
tm.tm_sec = seconds;
tm.tm_wday = 0;
tm.tm_yday = 0;
tm.tm_isdst = -1;
t = time(NULL);
if (*atz == '-' || *atz == '+') { /* numeric timezone */
time_t lt, gt;
time_t a;
struct tm ltt, gtt;
gint Hour = atoi(atz);
gint Min = atoi(atz+4);
if (Hour < 0)
Min = -Min;
tm.tm_hour -= Hour;
tm.tm_min -= Min;
tm.tm_isdst = 0;
localtime_r(&t, <t);
lt = mktime(<t);
gmtime_r(&t, >t);
gt = mktime(>t);
tm.tm_sec += lt - gt;
a = mktime(&tm);
return a;
} else if (*atz == 'Z' && *(atz+1) == '\0') { /* Z timezone */
time_t lt, gt;
time_t a;
struct tm ltt, gtt;
tm.tm_isdst = 0;
localtime_r(&t, <t);
lt = mktime(<t);
gmtime_r(&t, >t);
gt = mktime(>t);
tm.tm_sec += lt - gt;
a = mktime(&tm);
return a;
} else { /* named timezone */
int pid;
int fd[2];
char buf[101];
time_t a;
ssize_t size;
if (pipe(fd) == -1)
return 1073741824;
pid = fork();
switch (pid) {
case -1:
close(fd[0]);
close(fd[1]);
return 1073741824;
break;
case 0:
close(fd[0]);
setenv("TZ", atz, 1);
tzset();
a = mktime(&tm);
g_snprintf(buf, 100, "%d", (int)a);
full_write(fd[1], buf, strlen(buf));
close(fd[1]);
exit(0);
default:
close(fd[1]);
size = full_read(fd[0], buf, 100);
if (size < 0) size = 0;
close(fd[0]);
buf[size] = '\0';
waitpid(pid, NULL, 0);
break;
}
return atoi(buf);
}
}
}
static gboolean
is_non_empty_string(const char *str)
{
return str && str[0] != '\0';
}
static char *
build_url(
S3Handle *hdl,
const char *bucket,
const char *key,
const char *subresource,
const char **query)
{
GString *url = NULL;
char *esc_bucket = NULL, *esc_key = NULL;
if ((hdl->s3_api == S3_API_SWIFT_1 || hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3 || hdl->s3_api == S3_API_OAUTH2) &&
hdl->x_storage_url) {
url = g_string_new(hdl->x_storage_url);
g_string_append(url, "/");
} else {
/* scheme */
url = g_string_new("http");
if (hdl->use_ssl)
g_string_append(url, "s");
g_string_append(url, "://");
/* domain */
if (hdl->use_subdomain && bucket)
g_string_append_printf(url, "%s.%s", bucket, hdl->host);
else
g_string_append_printf(url, "%s", hdl->host);
if (hdl->service_path) {
g_string_append_printf(url, "%s/", hdl->service_path);
} else {
g_string_append(url, "/");
}
}
/* path */
if (!hdl->use_subdomain && bucket) {
/* curl_easy_escape addeded in 7.15.4 */
#if LIBCURL_VERSION_NUM >= 0x070f04
curl_version_info_data *info;
/* check the runtime version too */
info = curl_version_info(CURLVERSION_NOW);
if (info->version_num >= 0x070f04)
esc_bucket = curl_easy_escape(hdl->curl, bucket, 0);
else
esc_bucket = curl_escape(bucket, 0);
#else
esc_bucket = curl_escape(bucket, 0);
#endif
if (!esc_bucket) goto cleanup;
g_string_append_printf(url, "%s", esc_bucket);
if (key)
g_string_append(url, "/");
curl_free(esc_bucket);
}
if (key) {
/* curl_easy_escape addeded in 7.15.4 */
#if LIBCURL_VERSION_NUM >= 0x070f04
curl_version_info_data *info;
/* check the runtime version too */
info = curl_version_info(CURLVERSION_NOW);
if (info->version_num >= 0x070f04)
esc_key = curl_easy_escape(hdl->curl, key, 0);
else
esc_key = curl_escape(key, 0);
#else
esc_key = curl_escape(key, 0);
#endif
if (!esc_key) goto cleanup;
g_string_append_printf(url, "%s", esc_key);
curl_free(esc_key);
}
if (url->str[strlen(url->str)-1] == '/') {
g_string_truncate(url, strlen(url->str)-1);
}
/* query string */
if (subresource || query || (hdl->s3_api == S3_API_CASTOR && hdl->tenant_name))
g_string_append(url, "?");
if (subresource)
g_string_append(url, subresource);
if (subresource && query)
g_string_append(url, "&");
if (query) {
const char **q;
gboolean first = TRUE;
for (q = query; *q != NULL; q++) {
if (!first) {
g_string_append_c(url, '&');
}
g_string_append(url, *q);
first = FALSE;
}
}
/* add CAStor tenant domain override query arg */
if (hdl->s3_api == S3_API_CASTOR && hdl->tenant_name) {
if (subresource || query) {
g_string_append(url, "&");
}
g_string_append_printf(url, "domain=%s", hdl->tenant_name);
}
cleanup:
return g_string_free(url, FALSE);
}
static struct curl_slist *
authenticate_request(S3Handle *hdl,
const char *verb,
const char *bucket,
const char *key,
const char *subresource,
const char **query,
const char *md5_hash,
const char *data_SHA256Hash,
const char *content_type,
const size_t content_length,
const char *project_id)
{
time_t t;
struct tm tmp;
char *date = NULL;
char *szS3Date = NULL;
char *zulu_date = NULL;
char *buf = NULL;
#if (defined OPENSSL_VERSION_NUMBER && OPENSSL_VERSION_NUMBER < 0x10100000L) \
|| defined LIBRESSL_VERSION_NUMBER
HMAC_CTX ctx;
#else
HMAC_CTX *ctx;
#endif
GByteArray *md = NULL;
char *auth_base64 = NULL;
struct curl_slist *headers = NULL;
char *esc_bucket = NULL, *esc_key = NULL;
GString *auth_string = NULL;
char *reps = NULL;
/* From RFC 2616 */
static const char *wkday[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
static const char *month[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
/* calculate the date */
t = time(NULL);
/* sync clock with amazon s3 */
t = t + hdl->time_offset_with_s3;
#ifdef _WIN32
if (!gmtime_s(&tmp, &t)) g_debug("localtime error");
#else
if (!gmtime_r(&t, &tmp)) perror("localtime");
#endif
date = g_strdup_printf("%s, %02d %s %04d %02d:%02d:%02d GMT",
wkday[tmp.tm_wday], tmp.tm_mday, month[tmp.tm_mon], 1900+tmp.tm_year,
tmp.tm_hour, tmp.tm_min, tmp.tm_sec);
szS3Date = g_strdup_printf("%04d%02d%02d",1900+tmp.tm_year,tmp.tm_mon+1,tmp.tm_mday);
zulu_date = g_strdup_printf("%04d%02d%02dT%02d%02d%02dZ",
1900+tmp.tm_year, tmp.tm_mon+1, tmp.tm_mday,
tmp.tm_hour, tmp.tm_min, tmp.tm_sec);
if (hdl->s3_api == S3_API_SWIFT_1) {
if (!bucket) {
buf = g_strdup_printf("X-Auth-User: %s", hdl->swift_account_id);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("X-Auth-Key: %s", hdl->swift_access_key);
headers = curl_slist_append(headers, buf);
g_free(buf);
} else {
buf = g_strdup_printf("X-Auth-Token: %s", hdl->x_auth_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
} else if (hdl->s3_api == S3_API_SWIFT_2) {
if (bucket) {
buf = g_strdup_printf("X-Auth-Token: %s", hdl->x_auth_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("Accept: %s", "application/xml");
headers = curl_slist_append(headers, buf);
g_free(buf);
} else {
buf = g_strdup_printf("Accept: %s", "application/json");
headers = curl_slist_append(headers, buf);
g_free(buf);
}
} else if (hdl->s3_api == S3_API_SWIFT_3) {
if (bucket) {
buf = g_strdup_printf("X-Auth-Token: %s", hdl->x_auth_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("Accept: %s", "application/xml");
headers = curl_slist_append(headers, buf);
g_free(buf);
} else {
buf = g_strdup_printf("Accept: %s", "application/json");
headers = curl_slist_append(headers, buf);
g_free(buf);
}
} else if (hdl->s3_api == S3_API_OAUTH2) {
if (bucket) {
buf = g_strdup_printf("Authorization: Bearer %s", hdl->access_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
} else if (hdl->s3_api == S3_API_CASTOR) {
if (g_str_equal(verb, "PUT") || g_str_equal(verb, "POST")) {
if (key) {
buf = g_strdup("CAStor-Application: Amanda");
headers = curl_slist_append(headers, buf);
g_free(buf);
reps = g_strdup(hdl->reps); /* object replication level */
} else {
reps = g_strdup(hdl->reps_bucket); /* bucket replication level */
}
/* set object replicas in lifepoint */
buf = g_strdup_printf("lifepoint: [] reps=%s", reps);
headers = curl_slist_append(headers, buf);
g_free(buf);
g_free(reps);
}
} else if (hdl->s3_api == S3_API_AWS4) {
/* http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html */
/* http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html */
GString *strSignedHeaders = g_string_new("");
GString *string_to_sign;
char *canonical_hash = NULL;
unsigned char *strSecretKey = NULL;
unsigned char *signkey1 = NULL;
unsigned char *signkey2 = NULL;
unsigned char *signkey3 = NULL;
unsigned char *signingKey = NULL;
unsigned char *signature = NULL;
unsigned char *signatureHex = NULL;
/* verb */
auth_string = g_string_new(verb);
g_string_append(auth_string, "\n");
/* CanonicalizedResource */
g_string_append(auth_string, "/");
if (hdl->use_subdomain) {
if (key) {
char *esc_key = s3_uri_encode(key, 0);
g_string_append(auth_string, esc_key);
g_free(esc_key);
}
} else {
g_string_append(auth_string, bucket);
if (key) {
char *esc_key = s3_uri_encode(key, 0);
g_string_append(auth_string, "/");
g_string_append(auth_string, esc_key);
g_free(esc_key);
}
}
g_string_append(auth_string, "\n");
if (query) {
gboolean sub_done = !subresource;
gboolean first = TRUE;
const char **q;
for (q = query; *q != NULL; q++) {
if (!first) {
g_string_append_c(auth_string, '&');
}
if (!sub_done && strcmp(subresource, *q) < 0) {
g_string_append(auth_string, subresource);
g_string_append_c(auth_string, '=');
g_string_append_c(auth_string, '&');
sub_done = TRUE;
}
g_string_append(auth_string, *q);
first = FALSE;
}
if (!sub_done) {
g_string_append_c(auth_string, '&');
g_string_append(auth_string, subresource);
g_string_append_c(auth_string, '=');
}
} else if (subresource) {
g_string_append(auth_string, subresource);
g_string_append(auth_string, "=");
}
g_string_append(auth_string, "\n");
/* Header must be in alphebetic order */
if (hdl->use_subdomain) {
g_string_append(auth_string, "host:");
g_string_append(auth_string, bucket);
g_string_append(auth_string, ".");
g_string_append(auth_string, hdl->host_without_port);
g_string_append(auth_string, "\n");
} else {
g_string_append(auth_string, "host:");
g_string_append(auth_string, hdl->host_without_port);
g_string_append(auth_string, "\n");
}
g_string_append(strSignedHeaders, "host");
g_string_append(auth_string, "x-amz-content-sha256:");
g_string_append(auth_string, data_SHA256Hash);
g_string_append(auth_string, "\n");
g_string_append(strSignedHeaders, ";x-amz-content-sha256");
g_string_append(auth_string, "x-amz-date:");
g_string_append(auth_string, zulu_date);
g_string_append(auth_string, "\n");
g_string_append(strSignedHeaders, ";x-amz-date");
if (hdl->server_side_encryption_header &&
is_non_empty_string(hdl->server_side_encryption)) {
g_string_append(auth_string, AMAZON_SERVER_SIDE_ENCRYPTION_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->server_side_encryption);
g_string_append(auth_string, "\n");
g_string_append(strSignedHeaders, ";"AMAZON_SERVER_SIDE_ENCRYPTION_HEADER);
buf = g_strdup_printf(AMAZON_SERVER_SIDE_ENCRYPTION_HEADER ": %s",
hdl->server_side_encryption);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if ((!subresource || !g_strstr_len(subresource, -1, "uploadId")) &&
(!query || !query[0] || !g_strstr_len(query[0], -1, "uploadId")) &&
(!query || !query[1] || !g_strstr_len(query[1], -1, "uploadId")) &&
is_non_empty_string(hdl->storage_class)) {
g_string_append(auth_string, AMAZON_STORAGE_CLASS_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->storage_class);
g_string_append(auth_string, "\n");
g_string_append(strSignedHeaders, ";"AMAZON_STORAGE_CLASS_HEADER);
buf = g_strdup_printf(AMAZON_STORAGE_CLASS_HEADER ": %s",
hdl->storage_class);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
/* no more header */
g_string_append(auth_string, "\n");
g_string_append(auth_string, strSignedHeaders->str);
g_string_append(auth_string, "\n");
g_string_append(auth_string, data_SHA256Hash);
canonical_hash = s3_compute_sha256_hash((unsigned char *)auth_string->str, auth_string->len);
if (!hdl->bucket_location) {
hdl->bucket_location = g_strdup("us-east-1");
}
string_to_sign = g_string_new("AWS4-HMAC-SHA256\n");
g_string_append(string_to_sign, zulu_date);
g_string_append(string_to_sign, "\n");
g_string_append(string_to_sign, szS3Date);
g_string_append(string_to_sign, "/");
g_string_append(string_to_sign, hdl->bucket_location);
g_string_append(string_to_sign, "/s3/aws4_request");
g_string_append(string_to_sign, "\n");
g_string_append(string_to_sign, canonical_hash);
//Calculate the AWS Signature Version 4
strSecretKey = (unsigned char *)g_strdup_printf("AWS4%s", hdl->secret_key);
signkey1 = EncodeHMACSHA256(strSecretKey, strlen((char *)strSecretKey),
szS3Date, strlen(szS3Date));
signkey2 = EncodeHMACSHA256(signkey1, 32, hdl->bucket_location, strlen(hdl->bucket_location));
signkey3 = EncodeHMACSHA256(signkey2, 32, "s3", 2);
signingKey = EncodeHMACSHA256(signkey3, 32, "aws4_request", 12);
signature = EncodeHMACSHA256(signingKey, 32, string_to_sign->str, (int)string_to_sign->len);
signatureHex = s3_tohex(signature, 32);
buf = g_strdup_printf("x-amz-content-sha256: %s", data_SHA256Hash);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("x-amz-date: %s", zulu_date);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,SignedHeaders=%s,Signature=%s", hdl->access_key, szS3Date, hdl->bucket_location, strSignedHeaders->str, signatureHex);
headers = curl_slist_append(headers, buf);
g_free(buf);
if (hdl->verbose) {
g_debug("bucket: %s", bucket);
g_debug("auth_string->str: %s", auth_string->str);
g_debug("string_to_sign: %s", string_to_sign->str);
g_debug("strSignedHeaders: %s", strSignedHeaders->str);
g_debug("canonical_hash: %s", canonical_hash);
g_debug("strSecretKey: %s", strSecretKey);
g_debug("signatureHex: %s", signatureHex);
}
g_free(canonical_hash);
g_free(strSecretKey);
g_free(signkey1);
g_free(signkey2);
g_free(signkey3);
g_free(signingKey);
g_free(signature);
g_free(signatureHex);
md5_hash = NULL;
} else { /* hdl->s3_api == S3_API_S3 */
/* Build the string to sign, per the S3 spec.
* See: "Authenticating REST Requests" - API Version 2006-03-01 pg 58
* http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html
*/
/* verb */
auth_string = g_string_new(verb);
g_string_append(auth_string, "\n");
/* Content-MD5 header */
if (md5_hash)
g_string_append(auth_string, md5_hash);
g_string_append(auth_string, "\n");
if (content_type) {
g_string_append(auth_string, content_type);
}
g_string_append(auth_string, "\n");
/* Date */
g_string_append(auth_string, date);
g_string_append(auth_string, "\n");
/* CanonicalizedAmzHeaders, sorted lexicographically */
if (is_non_empty_string(hdl->user_token)) {
g_string_append(auth_string, AMAZON_SECURITY_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->user_token);
g_string_append(auth_string, ",");
g_string_append(auth_string, STS_PRODUCT_TOKEN);
g_string_append(auth_string, "\n");
}
/* CanonicalizedAmzHeaders, sorted lexicographically */
if (is_non_empty_string(hdl->session_token)) {
g_string_append(auth_string, AMAZON_SECURITY_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->session_token);
g_string_append(auth_string, "\n");
}
if (hdl->server_side_encryption_header &&
is_non_empty_string(hdl->server_side_encryption)) {
g_string_append(auth_string, AMAZON_SERVER_SIDE_ENCRYPTION_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->server_side_encryption);
g_string_append(auth_string, "\n");
}
if ((!subresource || !g_strstr_len(subresource, -1, "uploadId")) &&
hdl->s3_api == S3_API_S3 &&
is_non_empty_string(hdl->storage_class)) {
g_string_append(auth_string, AMAZON_STORAGE_CLASS_HEADER);
g_string_append(auth_string, ":");
g_string_append(auth_string, hdl->storage_class);
g_string_append(auth_string, "\n");
}
/* CanonicalizedResource */
if (hdl->service_path) {
g_string_append(auth_string, hdl->service_path);
}
g_string_append(auth_string, "/");
if (bucket) {
if (hdl->use_subdomain)
g_string_append(auth_string, bucket);
else {
esc_bucket = curl_escape(bucket, 0);
if (!esc_bucket) goto cleanup;
g_string_append(auth_string, esc_bucket);
}
}
if (bucket && (hdl->use_subdomain || key))
g_string_append(auth_string, "/");
if (key) {
esc_key = curl_escape(key, 0);
if (!esc_key) goto cleanup;
g_string_append(auth_string, esc_key);
}
if (subresource) {
g_string_append(auth_string, "?");
g_string_append(auth_string, subresource);
}
/* run HMAC-SHA1 on the canonicalized string */
md = g_byte_array_sized_new(EVP_MAX_MD_SIZE+1);
#if (defined OPENSSL_VERSION_NUMBER && OPENSSL_VERSION_NUMBER < 0x10100000L) \
|| defined LIBRESSL_VERSION_NUMBER
HMAC_CTX_init(&ctx);
HMAC_Init_ex(&ctx, hdl->secret_key, (int) strlen(hdl->secret_key),
EVP_sha1(), NULL);
HMAC_Update(&ctx, (unsigned char*) auth_string->str, auth_string->len);
HMAC_Final(&ctx, md->data, &md->len);
HMAC_CTX_cleanup(&ctx);
#else
ctx = HMAC_CTX_new();
HMAC_CTX_reset(ctx);
HMAC_Init_ex(ctx, hdl->secret_key, (int) strlen(hdl->secret_key),
EVP_sha1(), NULL);
HMAC_Update(ctx, (unsigned char*) auth_string->str, auth_string->len);
HMAC_Final(ctx, md->data, &md->len);
HMAC_CTX_free(ctx);
#endif
auth_base64 = s3_base64_encode(md);
/* append the new headers */
if (is_non_empty_string(hdl->user_token)) {
/* Devpay headers are included in hash. */
buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s",
hdl->user_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s",
STS_PRODUCT_TOKEN);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (is_non_empty_string(hdl->session_token)) {
/* Devpay headers are included in hash. */
buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s",
hdl->session_token);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (hdl->server_side_encryption_header &&
is_non_empty_string(hdl->server_side_encryption)) {
buf = g_strdup_printf(AMAZON_SERVER_SIDE_ENCRYPTION_HEADER ": %s",
hdl->server_side_encryption);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if ((!subresource || !g_strstr_len(subresource, -1, "uploadId")) &&
hdl->s3_api != S3_API_OAUTH2 &&
is_non_empty_string(hdl->storage_class)) {
buf = g_strdup_printf(AMAZON_STORAGE_CLASS_HEADER ": %s",
hdl->storage_class);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
/* Remove Content-Type header */
buf = g_strdup("Content-Type:");
headers = curl_slist_append(headers, buf);
g_free(buf);
buf = g_strdup_printf("Authorization: AWS %s:%s",
hdl->access_key, auth_base64);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (md5_hash && '\0' != md5_hash[0]) {
buf = g_strdup_printf("Content-MD5: %s", md5_hash);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (content_length > 0) {
buf = g_strdup_printf("Content-Length: %zu", content_length);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (content_type) {
buf = g_strdup_printf("Content-Type: %s", content_type);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (hdl->s3_api == S3_API_OAUTH2) {
buf = g_strdup_printf("x-goog-api-version: 2");
headers = curl_slist_append(headers, buf);
g_free(buf);
}
if (project_id && hdl->s3_api == S3_API_OAUTH2) {
buf = g_strdup_printf("x-goog-project-id: %s", project_id);
headers = curl_slist_append(headers, buf);
g_free(buf);
}
buf = g_strdup_printf("Date: %s", date);
headers = curl_slist_append(headers, buf);
g_free(buf);
cleanup:
g_free(date);
g_free(szS3Date);
g_free(esc_bucket);
g_free(esc_key);
if (md) g_byte_array_free(md, TRUE);
g_free(auth_base64);
if (auth_string) g_string_free(auth_string, TRUE);
return headers;
}
/* Functions for a SAX parser to parse the XML failure from Amazon */
/* Private structure for our "thunk", which tracks where the user is in the list
* * of keys. */
struct failure_thunk {
gboolean want_text;
gboolean in_title;
gboolean in_body;
gboolean in_code;
gboolean in_message;
gboolean in_details;
gboolean in_access;
gboolean in_token;
gboolean in_serviceCatalog;
gboolean in_service;
gboolean in_endpoint;
gboolean in_uploadId;
gint in_others;
gchar *text;
gsize text_len;
gchar *message;
gchar *details;
gchar *error_name;
gchar *token_id;
gchar *service_type;
gchar *service_public_url;
gint64 expires;
gchar *uploadId;
gchar *bucket_location;
};
static void
failure_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
const gchar **attribute_names,
const gchar **attribute_values,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct failure_thunk *thunk = (struct failure_thunk *)user_data;
const gchar **att_name, **att_value;
if (g_ascii_strcasecmp(element_name, "title") == 0) {
thunk->in_title = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "body") == 0) {
thunk->in_body = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "code") == 0) {
thunk->in_code = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "message") == 0) {
thunk->in_message = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "details") == 0) {
thunk->in_details = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "access") == 0) {
thunk->in_access = 1;
thunk->in_others = 0;
} else if (g_ascii_strcasecmp(element_name, "token") == 0) {
thunk->in_token = 1;
thunk->in_others = 0;
for (att_name=attribute_names, att_value=attribute_values;
*att_name != NULL;
att_name++, att_value++) {
if (g_str_equal(*att_name, "id")) {
thunk->token_id = g_strdup(*att_value);
}
if (g_str_equal(*att_name, "expires") && strlen(*att_value) >= 19) {
thunk->expires = rfc3339_date(*att_value) - 600;
}
}
} else if (g_ascii_strcasecmp(element_name, "serviceCatalog") == 0) {
thunk->in_serviceCatalog = 1;
thunk->in_others = 0;
} else if (g_ascii_strcasecmp(element_name, "service") == 0) {
thunk->in_service = 1;
thunk->in_others = 0;
for (att_name=attribute_names, att_value=attribute_values;
*att_name != NULL;
att_name++, att_value++) {
if (g_str_equal(*att_name, "type")) {
thunk->service_type = g_strdup(*att_value);
}
}
} else if (g_ascii_strcasecmp(element_name, "endpoint") == 0) {
thunk->in_endpoint = 1;
thunk->in_others = 0;
if (thunk->service_type &&
g_str_equal(thunk->service_type, "object-store")) {
char *service_public_url = NULL;
char *region = NULL;
for (att_name=attribute_names, att_value=attribute_values;
*att_name != NULL;
att_name++, att_value++) {
if (g_str_equal(*att_name, "publicURL")) {
service_public_url = g_strdup(*att_value);
}
if (g_str_equal(*att_name, "region")) {
region = g_strdup(*att_value);
}
}
if (region && service_public_url) {
if (!thunk->bucket_location ||
strcmp(thunk->bucket_location, region) == 0) {
thunk->service_public_url = service_public_url;
} else {
g_free(service_public_url);
}
} else {
thunk->service_public_url = service_public_url;
}
g_free(region);
}
} else if (g_ascii_strcasecmp(element_name, "error") == 0) {
for (att_name=attribute_names, att_value=attribute_values;
*att_name != NULL;
att_name++, att_value++) {
if (g_str_equal(*att_name, "message")) {
thunk->message = g_strdup(*att_value);
}else if (g_str_equal(*att_name, "title")) {
thunk->error_name = g_strdup(*att_value);
// hdl->last_s3_error_code = s3_error_code_from_name(thunk->error_name);
}
}
} else if (g_ascii_strcasecmp(element_name, "uploadid") == 0) {
thunk->in_uploadId = 1;
thunk->in_others = 0;
thunk->want_text = 1;
} else {
thunk->in_others++;
}
}
static void
failure_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct failure_thunk *thunk = (struct failure_thunk *)user_data;
if (g_ascii_strcasecmp(element_name, "title") == 0) {
char *p = strchr(thunk->text, ' ');
if (p) {
p++;
if (*p) {
thunk->error_name = g_strdup(p);
}
}
g_free(thunk->text);
thunk->text = NULL;
thunk->in_title = 0;
} else if (g_ascii_strcasecmp(element_name, "body") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
g_strstrip(thunk->message);
thunk->text = NULL;
thunk->in_body = 0;
} else if (g_ascii_strcasecmp(element_name, "code") == 0) {
g_free(thunk->error_name);
thunk->error_name = thunk->text;
thunk->text = NULL;
thunk->in_code = 0;
} else if (g_ascii_strcasecmp(element_name, "message") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
thunk->in_message = 0;
} else if (g_ascii_strcasecmp(element_name, "details") == 0) {
g_free(thunk->details);
thunk->details = thunk->text;
thunk->text = NULL;
thunk->in_details = 0;
} else if (g_ascii_strcasecmp(element_name, "access") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
thunk->in_access = 0;
} else if (g_ascii_strcasecmp(element_name, "token") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
thunk->in_token = 0;
} else if (g_ascii_strcasecmp(element_name, "serviceCatalog") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
thunk->in_serviceCatalog = 0;
} else if (g_ascii_strcasecmp(element_name, "service") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
g_free(thunk->service_type);
thunk->service_type = NULL;
thunk->in_service = 0;
} else if (g_ascii_strcasecmp(element_name, "endpoint") == 0) {
g_free(thunk->message);
thunk->message = thunk->text;
thunk->text = NULL;
thunk->in_endpoint = 0;
} else if (g_ascii_strcasecmp(element_name, "uploadid") == 0) {
g_free(thunk->uploadId);
thunk->uploadId = thunk->text;
thunk->text = NULL;
thunk->in_uploadId = 0;
} else {
thunk->in_others--;
g_free(thunk->text);
thunk->text = NULL;
}
}
static void
failure_text(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *text,
gsize text_len,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct failure_thunk *thunk = (struct failure_thunk *)user_data;
if (thunk->want_text && thunk->in_others == 0) {
char *new_text;
new_text = g_strndup(text, text_len);
if (thunk->text) {
strappend(thunk->text, new_text);
g_free(new_text);
} else {
thunk->text = new_text;
}
}
}
static void
parse_swift_v2_endpoints(
gpointer data,
gpointer user_data)
{
amjson_t *json = data;
struct failure_thunk *pthunk = user_data;
assert(json);
if (get_json_type(json) == JSON_HASH) {
amjson_t *endpoint_region = get_json_hash_from_key(json, "region");
amjson_t *endpoint_publicURL = get_json_hash_from_key(json, "publicURL");
char *region = NULL;
char *service_public_url = NULL;
if (endpoint_region && get_json_type(endpoint_region) == JSON_STRING) {
region = get_json_string(endpoint_region);
}
if (endpoint_publicURL && get_json_type(endpoint_publicURL) == JSON_STRING) {
service_public_url = get_json_string(endpoint_publicURL);
}
if (region && service_public_url) {
if (!pthunk->bucket_location ||
strcmp(pthunk->bucket_location, region) == 0) {
pthunk->service_public_url = g_strdup(service_public_url);
}
} else {
pthunk->service_public_url = g_strdup(service_public_url);
}
}
}
static void
parse_swift_v2_serviceCatalog(
gpointer data,
gpointer user_data)
{
amjson_t *json = data;
assert(json);
if (get_json_type(json) == JSON_HASH) {
amjson_t *catalog_type = get_json_hash_from_key(json, "type");
if (get_json_type(catalog_type) == JSON_STRING && g_str_equal(get_json_string(catalog_type), "object-store")) {
amjson_t *catalog_endpoints = get_json_hash_from_key(json, "endpoints");
if (get_json_type(catalog_endpoints) == JSON_ARRAY) {
foreach_json_array(catalog_endpoints, parse_swift_v2_endpoints, user_data);
}
}
}
}
static void
parse_swift_v3_endpoints(
gpointer data,
gpointer user_data)
{
amjson_t *json = data;
struct failure_thunk *pthunk = user_data;
assert(json);
if (get_json_type(json) == JSON_HASH) {
amjson_t *endpoint_region = get_json_hash_from_key(json, "region_id");
amjson_t *endpoint_interface = get_json_hash_from_key(json, "interface");
amjson_t *endpoint_url = get_json_hash_from_key(json, "url");
char *region = NULL;
char *service_public_url = NULL;
if (endpoint_region && get_json_type(endpoint_region) == JSON_STRING) {
region = get_json_string(endpoint_region);
}
if (endpoint_interface && get_json_type(endpoint_interface) == JSON_STRING) {
char *interface = get_json_string(endpoint_interface);
if (g_str_equal(interface, "public")) {
if (endpoint_url && get_json_type(endpoint_url) == JSON_STRING) {
service_public_url = get_json_string(endpoint_url);
}
}
}
if (region && service_public_url) {
if (!pthunk->bucket_location ||
strcmp(pthunk->bucket_location, region) == 0) {
pthunk->service_public_url = g_strdup(service_public_url);
}
} else if (!pthunk->service_public_url && service_public_url) {
pthunk->service_public_url = g_strdup(service_public_url);
}
}
}
static void
parse_swift_v3_catalog(
gpointer data,
gpointer user_data)
{
amjson_t *json = data;
assert(json);
if (get_json_type(json) == JSON_HASH) {
amjson_t *catalog_type = get_json_hash_from_key(json, "type");
if (get_json_type(catalog_type) == JSON_STRING && g_str_equal(get_json_string(catalog_type), "object-store")) {
amjson_t *catalog_endpoints = get_json_hash_from_key(json, "endpoints");
if (get_json_type(catalog_endpoints) == JSON_ARRAY) {
foreach_json_array(catalog_endpoints, parse_swift_v3_endpoints, user_data);
}
}
}
}
static gboolean
interpret_response(S3Handle *hdl,
CURLcode curl_code,
char *curl_error_buffer,
gchar *body,
guint body_len,
const char *etag,
const char *content_md5)
{
long response_code = 0;
gboolean ret = TRUE;
struct failure_thunk thunk;
GMarkupParseContext *ctxt = NULL;
static GMarkupParser parser = { failure_start_element, failure_end_element, failure_text, NULL, NULL };
GError *err = NULL;
if (!hdl) return FALSE;
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = NULL;
hdl->last_response_code = 0;
hdl->last_s3_error_code = 0;
hdl->last_curl_code = 0;
curl_easy_getinfo(hdl->curl, CURLINFO_RESPONSE_CODE, &response_code);
hdl->last_response_code = response_code;
/* bail out from a CURL error */
if (curl_code != CURLE_OK) {
hdl->last_curl_code = curl_code;
hdl->last_message = g_strdup_printf("CURL error: %s", curl_error_buffer);
if (response_code == 0)
return FALSE;
}
/* check ETag, if present and not CAStor */
if (etag && content_md5 && 200 == response_code &&
hdl->s3_api != S3_API_CASTOR) {
if (etag && g_ascii_strcasecmp(etag, content_md5))
hdl->last_message = g_strdup("S3 Error: Possible data corruption (ETag returned by Amazon did not match the MD5 hash of the data sent)");
else
ret = FALSE;
return ret;
}
/* Now look at the body to try to get the actual Amazon error message. */
/* impose a reasonable limit on body size */
if (body_len > MAX_ERROR_RESPONSE_LEN) {
hdl->last_message = g_strdup("S3 Error: Unknown (response body too large to parse)");
return FALSE;
} else if (!body || body_len == 0) {
if (response_code < 100 || response_code >= 400) {
hdl->last_message =
g_strdup("S3 Error: Unknown (empty response body)");
return TRUE; /* perhaps a network error; retry the request */
} else {
/* 2xx and 3xx codes without body are good result */
hdl->last_s3_error_code = S3_ERROR_None;
return FALSE;
}
}
if (hdl->verbose) {
char *body_copy = g_strndup(body, body_len);
g_debug("data in %d: %s", (int)body_len, body_copy);
amfree(body_copy);
}
thunk.in_title = FALSE;
thunk.in_body = FALSE;
thunk.in_code = FALSE;
thunk.in_message = FALSE;
thunk.in_details = FALSE;
thunk.in_access = FALSE;
thunk.in_token = FALSE;
thunk.in_serviceCatalog = FALSE;
thunk.in_service = FALSE;
thunk.in_endpoint = FALSE;
thunk.in_uploadId = FALSE;
thunk.in_others = 0;
thunk.text = NULL;
thunk.want_text = FALSE;
thunk.text_len = 0;
thunk.message = NULL;
thunk.details = NULL;
thunk.error_name = NULL;
thunk.token_id = NULL;
thunk.service_type = NULL;
thunk.service_public_url = NULL;
thunk.expires = 0;
thunk.uploadId = NULL;
thunk.bucket_location = hdl->bucket_location;
if ((hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2) &&
hdl->content_type &&
(g_str_equal(hdl->content_type, "text/html") ||
g_str_equal(hdl->content_type, "text/plain"))) {
char *body_copy = g_strndup(body, body_len);
char *b = body_copy;
char *p = strchr(b, '\n');
char *p1;
if (p) { /* first line: error code */
*p = '\0';
p++;
p1 = strchr(b, ' ');
if (p1) {
p1++;
if (*p1) {
thunk.error_name = g_strdup(p1);
}
}
b = p;
}
p = strchr(b, '\n');
if (p) { /* second line: error message */
*p = '\0';
p++;
thunk.message = g_strdup(p);
g_strstrip(thunk.message);
b = p;
}
goto parsing_done;
} else if ((hdl->s3_api == S3_API_SWIFT_3) &&
hdl->content_type &&
(g_str_equal(hdl->content_type, "text/html") ||
g_str_equal(hdl->content_type, "text/plain"))) {
char *body_copy = g_strndup(body, body_len);
regmatch_t pmatch[2];
if (!s3_regexec_wrap(&html_error_name_regex, body_copy, 2, pmatch, 0)) {
thunk.error_name = find_regex_substring(body_copy, pmatch[1]);
am_strrmspace(thunk.error_name);
}
if (!s3_regexec_wrap(&html_message_regex, body_copy, 2, pmatch, 0)) {
thunk.message = find_regex_substring(body_copy, pmatch[1]);
}
goto parsing_done;
} else if ((hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3) &&
hdl->content_type &&
g_str_equal(hdl->content_type, "application/json")) {
char *body_copy = g_strndup(body, body_len);
char *code = NULL;
char *details = NULL;
regmatch_t pmatch[2];
if (hdl->getting_swift_2_token) {
amjson_t *json = parse_json(body_copy);
if (get_json_type(json) == JSON_HASH) {
amjson_t *json_access = get_json_hash_from_key(json, "access");
if (json_access && get_json_type(json_access) == JSON_HASH) {
amjson_t *json_token = get_json_hash_from_key(json_access, "token");
amjson_t *json_catalog = get_json_hash_from_key(json_access, "serviceCatalog");
if (json_token && get_json_type(json_token) == JSON_HASH) {
amjson_t *json_token_id = get_json_hash_from_key(json_token, "id");
amjson_t *json_token_expires = get_json_hash_from_key(json_token, "expires");
if (json_token_id && get_json_type(json_token_id) == JSON_STRING) {
thunk.token_id = g_strdup(get_json_string(json_token_id));
}
if (json_token_expires && get_json_type(json_token_expires) == JSON_STRING) {
thunk.expires = rfc3339_date(get_json_string(json_token_expires));
}
}
if (json_catalog && get_json_type(json_catalog) == JSON_ARRAY) {
foreach_json_array(json_catalog, parse_swift_v2_serviceCatalog, &thunk);
}
if (thunk.token_id && thunk.expires && thunk.service_public_url) {
g_free(body_copy);
goto parse_done;
}
}
}
} else if (hdl->getting_swift_3_token) {
amjson_t *json = parse_json(body_copy);
// g_debug("json: %s", json_to_string(json));
if (get_json_type(json) == JSON_HASH) {
amjson_t *json_token = get_json_hash_from_key(json, "token");
if (json_token && get_json_type(json_token) == JSON_HASH) {
amjson_t *json_catalog = get_json_hash_from_key(json_token, "catalog");
amjson_t *json_expires_at = get_json_hash_from_key(json_token, "expires_at");
if (json_catalog && get_json_type(json_catalog) == JSON_ARRAY) {
foreach_json_array(json_catalog, parse_swift_v3_catalog, &thunk);
}
if (json_expires_at && get_json_type(json_expires_at) == JSON_STRING) {
thunk.expires = rfc3339_date(get_json_string(json_expires_at));
}
if (thunk.expires && thunk.service_public_url) {
g_free(body_copy);
goto parse_done;
}
}
}
}
if (!s3_regexec_wrap(&code_regex, body_copy, 2, pmatch, 0)) {
code = find_regex_substring(body_copy, pmatch[1]);
}
if (!s3_regexec_wrap(&details_regex, body_copy, 2, pmatch, 0)) {
details = find_regex_substring(body_copy, pmatch[1]);
}
if (!details) {
if (!s3_regexec_wrap(&json_message_regex, body_copy, 2, pmatch, 0)) {
details = find_regex_substring(body_copy, pmatch[1]);
}
}
if (code && details) {
hdl->last_message = g_strdup_printf("%s (%s)", details, code);
} else if (code) {
hdl->last_message = g_strdup_printf("(%s)", code);
} else if (details) {
hdl->last_message = g_strdup_printf("%s", details);
} else {
hdl->last_message = NULL;
}
g_free(code);
g_free(details);
g_free(body_copy);
return FALSE;
} else if (hdl->s3_api == S3_API_CASTOR) {
/* The error mesage is the body */
hdl->last_message = g_strndup(body, body_len);
return FALSE;
} else if ((hdl->content_type &&
!g_str_equal(hdl->content_type, "application/xml")) ||
(!hdl->content_type &&
!g_str_equal(hdl->transfer_encoding, "chunked") &&
!g_strstr_len(body, body_len, "xml version"))) {
return FALSE;
}
/* run the parser over it */
ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
if (!g_markup_parse_context_parse(ctxt, body, body_len, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
goto cleanup;
}
if (!g_markup_parse_context_end_parse(ctxt, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
goto cleanup;
}
g_markup_parse_context_free(ctxt);
ctxt = NULL;
parse_done:
if (hdl->s3_api == S3_API_SWIFT_2) {
if (!hdl->x_auth_token && thunk.token_id) {
hdl->x_auth_token = thunk.token_id;
g_debug("x_auth_token: %s", hdl->x_auth_token);
thunk.token_id = NULL;
}
if (!hdl->x_storage_url && thunk.service_public_url) {
hdl->x_storage_url = thunk.service_public_url;
g_debug("x_storage_url: %s", hdl->x_storage_url);
thunk.service_public_url = NULL;
} else if (!thunk.message && !thunk.error_name) {
if (!hdl->x_storage_url && !thunk.service_public_url) {
if (thunk.bucket_location) {
thunk.message = g_strdup_printf("Did not find the publicURL for the '%s' region", thunk.bucket_location);
} else {
thunk.message = g_strdup_printf("Did not find the publicURL");
}
thunk.error_name =g_strdup("RegionNotFound");
}
}
} else if (hdl->s3_api == S3_API_SWIFT_3) {
if (!hdl->x_storage_url && thunk.service_public_url) {
hdl->x_storage_url = thunk.service_public_url;
g_debug("x_storage_url: %s", hdl->x_storage_url);
thunk.service_public_url = NULL;
} else if (!thunk.message && !thunk.error_name) {
if (!hdl->x_storage_url && !thunk.service_public_url) {
if (thunk.bucket_location) {
thunk.message = g_strdup_printf("Did not find the public url for the '%s' region", thunk.bucket_location);
} else {
thunk.message = g_strdup_printf("Did not find the public url");
}
thunk.error_name =g_strdup("RegionNotFound");
}
}
}
if (thunk.expires > 0) {
hdl->expires = thunk.expires;
}
parsing_done:
if (thunk.error_name) {
hdl->last_s3_error_code = s3_error_code_from_name(thunk.error_name);
g_free(thunk.error_name);
thunk.error_name = NULL;
}
if (thunk.message) {
g_free(hdl->last_message);
if (thunk.details) {
hdl->last_message = g_strdup_printf("%s: %s", thunk.message,
thunk.details);
amfree(thunk.message);
amfree(thunk.details);
} else {
hdl->last_message = thunk.message;
thunk.message = NULL; /* steal the reference to the string */
}
} else if (thunk.details) {
hdl->last_message = thunk.details;
thunk.details = NULL; /* steal the reference to the string */
}
if (thunk.uploadId) {
g_free(hdl->uploadId);
hdl->uploadId = thunk.uploadId;
thunk.uploadId = NULL; /* steal the reference to the string */
}
cleanup:
g_free(thunk.text);
g_free(thunk.message);
g_free(thunk.details);
g_free(thunk.error_name);
g_free(thunk.token_id);
g_free(thunk.service_type);
g_free(thunk.service_public_url);
g_free(thunk.uploadId);
return FALSE;
}
/* a CURLOPT_READFUNCTION to read data from a buffer. */
size_t
s3_buffer_read_func(void *ptr, size_t size, size_t nmemb, void * stream)
{
CurlBuffer *data = stream;
guint bytes_desired = (guint) size * nmemb;
guint avail;
if (data->mutex) { /* chunked transfer */
g_mutex_lock(data->mutex);
while (1) {
if (data->buffer_len == data->buffer_pos) {
avail = 0;
} else if (data->buffer_len > data->buffer_pos) {
avail = data->buffer_len - data->buffer_pos;
} else {
avail = data->max_buffer_size - data->buffer_pos + data->buffer_len;
}
if (avail > bytes_desired || data->end_of_buffer)
break;
g_cond_wait(data->cond, data->mutex);
}
if (bytes_desired > avail)
bytes_desired = avail;
if (bytes_desired > 0) {
if (data->buffer_len > data->buffer_pos) {
memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
data->buffer_pos += bytes_desired;
} else {
guint count_end = data->max_buffer_size - data->buffer_pos;
guint count_begin;
if (count_end > bytes_desired)
count_end = bytes_desired;
memcpy((char *)ptr, data->buffer + data->buffer_pos, count_end);
data->buffer_pos += count_end;
count_begin = bytes_desired - count_end;
if (count_begin > 0) {
memcpy((char *)ptr + count_end, data->buffer, count_begin);
data->buffer_pos = count_begin;
}
}
}
/* signal to add more data to the buffer */
g_cond_broadcast(data->cond);
g_mutex_unlock(data->mutex);
} else {
/* check the number of bytes remaining, just to be safe */
if (bytes_desired > data->buffer_len - data->buffer_pos)
bytes_desired = data->buffer_len - data->buffer_pos;
memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
data->buffer_pos += bytes_desired;
}
return bytes_desired;
}
size_t
s3_buffer_size_func(void *stream)
{
CurlBuffer *data = stream;
return data->buffer_len;
}
GByteArray*
s3_buffer_md5_func(void *stream)
{
CurlBuffer *data = stream;
GByteArray req_body_gba = {(guint8 *)data->buffer, data->buffer_len};
return s3_compute_md5_hash(&req_body_gba);
}
void
s3_buffer_reset_func(void *stream)
{
CurlBuffer *data = stream;
data->buffer_pos = 0;
}
/* a CURLOPT_WRITEFUNCTION to write data to a buffer. */
size_t
s3_buffer_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
{
CurlBuffer * data = stream;
guint new_bytes = (guint) size * nmemb;
guint bytes_needed = data->buffer_pos + new_bytes;
if (data->mutex) { /* chunked transfer */
g_mutex_lock(data->mutex);
/* error out if 2 block can't fit in the buffer */
if (new_bytes*2 > data->max_buffer_size) {
g_mutex_unlock(data->mutex);
return 0;
}
// Check for enough space in the buffer
while (1) {
guint avail;
if (data->buffer_len == data->buffer_pos) {
avail = data->max_buffer_size;
} else if (data->buffer_len > data->buffer_pos) {
avail = data->buffer_pos + data->max_buffer_size - data->buffer_len;
} else {
avail = data->buffer_pos - data->buffer_len;
}
if (avail > new_bytes) {
break;
}
g_cond_wait(data->cond, data->mutex);
}
// Copy the new data to the buffer
if (data->buffer_len > data->buffer_pos) {
guint count_end = data->max_buffer_size - data->buffer_len;
guint count_begin;
if (count_end > new_bytes)
count_end = new_bytes;
memcpy(data->buffer + data->buffer_len, ptr, count_end);
data->buffer_len += count_end;
count_begin = new_bytes - count_end;
if (count_begin > 0) {
memcpy(data->buffer, ptr + count_end, count_begin);
data->buffer_len = count_begin;
}
} else {
memcpy(data->buffer + data->buffer_len, ptr, new_bytes);
data->buffer_len += new_bytes;
}
g_cond_broadcast(data->cond);
g_mutex_unlock(data->mutex);
return new_bytes;
}
/* error out if the new size is greater than the maximum allowed */
if (data->max_buffer_size && bytes_needed > data->max_buffer_size)
return 0;
/* reallocate if necessary. We use exponential sizing to make this
* happen less often. */
if (bytes_needed > data->buffer_len) {
guint new_size = MAX(bytes_needed, data->buffer_len * 2);
if (data->max_buffer_size) {
new_size = MIN(new_size, data->max_buffer_size);
}
data->buffer = g_realloc(data->buffer, new_size);
data->buffer_len = new_size;
}
if (!data->buffer)
return 0; /* returning zero signals an error to libcurl */
/* actually copy the data to the buffer */
memcpy(data->buffer + data->buffer_pos, ptr, new_bytes);
data->buffer_pos += new_bytes;
/* signal success to curl */
return new_bytes;
}
/* a CURLOPT_READFUNCTION that writes nothing. */
size_t
s3_empty_read_func(G_GNUC_UNUSED void *ptr, G_GNUC_UNUSED size_t size, G_GNUC_UNUSED size_t nmemb, G_GNUC_UNUSED void * stream)
{
return 0;
}
size_t
s3_empty_size_func(G_GNUC_UNUSED void *stream)
{
return 0;
}
GByteArray*
s3_empty_md5_func(G_GNUC_UNUSED void *stream)
{
static const GByteArray empty = {(guint8 *) "", 0};
return s3_compute_md5_hash(&empty);
}
/* a CURLOPT_WRITEFUNCTION to write data that just counts data.
* s3_write_data should be NULL or a pointer to an gint64.
*/
size_t
s3_counter_write_func(G_GNUC_UNUSED void *ptr, size_t size, size_t nmemb, void *stream)
{
gint64 *count = (gint64*) stream, inc = nmemb*size;
if (count) *count += inc;
return inc;
}
void
s3_counter_reset_func(void *stream)
{
gint64 *count = (gint64*) stream;
if (count) *count = 0;
}
#ifdef _WIN32
/* a CURLOPT_READFUNCTION to read data from a file. */
size_t
s3_file_read_func(void *ptr, size_t size, size_t nmemb, void * stream)
{
HANDLE *hFile = (HANDLE *) stream;
DWORD bytes_read;
ReadFile(hFile, ptr, (DWORD) size*nmemb, &bytes_read, NULL);
return bytes_read;
}
size_t
s3_file_size_func(void *stream)
{
HANDLE *hFile = (HANDLE *) stream;
DWORD size = GetFileSize(hFile, NULL);
if (INVALID_FILE_SIZE == size) {
return -1;
} else {
return size;
}
}
GByteArray*
s3_file_md5_func(void *stream)
{
#define S3_MD5_BUF_SIZE (10*1024)
HANDLE *hFile = (HANDLE *) stream;
guint8 buf[S3_MD5_BUF_SIZE];
DWORD bytes_read;
MD5_CTX md5_ctx;
GByteArray *ret = NULL;
g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
ret = g_byte_array_sized_new(S3_MD5_HASH_BYTE_LEN);
g_byte_array_set_size(ret, S3_MD5_HASH_BYTE_LEN);
MD5_Init(&md5_ctx);
while (ReadFile(hFile, buf, S3_MD5_BUF_SIZE, &bytes_read, NULL)) {
MD5_Update(&md5_ctx, buf, bytes_read);
}
MD5_Final(ret->data, &md5_ctx);
g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
return ret;
#undef S3_MD5_BUF_SIZE
}
GByteArray*
s3_file_reset_func(void *stream)
{
g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
}
/* a CURLOPT_WRITEFUNCTION to write data to a file. */
size_t
s3_file_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
{
HANDLE *hFile = (HANDLE *) stream;
DWORD bytes_written;
WriteFile(hFile, ptr, (DWORD) size*nmemb, &bytes_written, NULL);
return bytes_written;
}
#endif
static int
curl_debug_message(CURL *curl G_GNUC_UNUSED,
curl_infotype type,
char *s,
size_t len,
void *unused G_GNUC_UNUSED)
{
char *lineprefix;
char *message;
char **lines, **line;
size_t i;
switch (type) {
case CURLINFO_TEXT:
lineprefix="";
break;
case CURLINFO_HEADER_IN:
lineprefix="Hdr In: ";
break;
case CURLINFO_HEADER_OUT:
lineprefix="Hdr Out: ";
break;
case CURLINFO_DATA_IN:
if (len > 3000) return 0;
for (i=0;i<len;i++) {
if (!g_ascii_isprint(s[i])) {
return 0;
}
}
lineprefix="Data In: ";
break;
case CURLINFO_DATA_OUT:
if (len > 3000) return 0;
for (i=0;i<len;i++) {
if (!g_ascii_isprint(s[i])) {
return 0;
}
}
lineprefix="Data Out: ";
break;
default:
/* ignore data in/out -- nobody wants to see that in the
* debug logs! */
return 0;
}
/* split the input into lines */
message = g_strndup(s, (gsize) len);
lines = g_strsplit(message, "\n", -1);
g_free(message);
for (line = lines; *line; line++) {
if (**line == '\0') continue; /* skip blank lines */
g_debug("%s%s", lineprefix, *line);
}
g_strfreev(lines);
return 0;
}
static s3_result_t
perform_request(S3Handle *hdl,
const char *verb,
const char *bucket,
const char *key,
const char *subresource,
const char **query,
const char *content_type,
const char *project_id,
struct curl_slist *user_headers,
s3_read_func read_func,
s3_reset_func read_reset_func,
s3_size_func size_func,
s3_md5_func md5_func,
gpointer read_data,
s3_write_func write_func,
s3_reset_func write_reset_func,
gpointer write_data,
s3_progress_func progress_func,
gpointer progress_data,
const result_handling_t *result_handling,
gboolean chunked)
{
char *url = NULL;
s3_result_t result = S3_RESULT_FAIL; /* assume the worst.. */
CURLcode curl_code = CURLE_OK;
char curl_error_buffer[CURL_ERROR_SIZE] = "";
struct curl_slist *headers = NULL;
/* Set S3Internal Data */
S3InternalData int_writedata = {{NULL, 0, 0, MAX_ERROR_RESPONSE_LEN, TRUE, NULL, NULL}, NULL, NULL, NULL, FALSE, FALSE, NULL, hdl};
gboolean should_retry;
gint retries = 0;
gint retry_after_close = 0;
gulong backoff = EXPONENTIAL_BACKOFF_START_USEC;
/* corresponds to PUT, HEAD, GET, and POST */
int curlopt_upload = 0, curlopt_nobody = 0, curlopt_httpget = 0, curlopt_post = 0;
/* do we want to examine the headers */
const char *curlopt_customrequest = NULL;
/* for MD5 calculation */
GByteArray *md5_hash = NULL;
gchar *md5_hash_hex = NULL, *md5_hash_b64 = NULL;
size_t request_body_size = 0;
char *data_SHA256Hash = NULL;
g_assert(hdl != NULL && hdl->curl != NULL);
if (hdl->s3_api == S3_API_OAUTH2 && !hdl->getting_oauth2_access_token &&
(!hdl->access_token || hdl->expires < time(NULL))) {
result = oauth2_get_access_token(hdl);
if (!result) {
g_debug("oauth2_get_access_token returned %d", result);
return result;
}
} else if (hdl->s3_api == S3_API_SWIFT_2 && !hdl->getting_swift_2_token &&
(!hdl->x_auth_token || hdl->expires < time(NULL))) {
result = get_openstack_swift_api_v2_setting(hdl);
if (!result) {
g_debug("get_openstack_swift_api_v2_setting returned %d", result);
return result;
}
} else if (hdl->s3_api == S3_API_SWIFT_3 && !hdl->getting_swift_3_token &&
(!hdl->x_auth_token || hdl->expires < time(NULL))) {
result = get_openstack_swift_api_v3_setting(hdl);
if (!result) {
g_debug("get_openstack_swift_api_v3_setting returned %d", result);
return result;
}
}
s3_reset(hdl);
url = build_url(hdl, bucket, key, subresource, query);
if (!url) goto cleanup;
/* libcurl may behave strangely if these are not set correctly */
if (g_str_has_prefix(verb, "PUT")) {
curlopt_upload = 1;
} else if (g_str_has_prefix(verb, "GET")) {
curlopt_httpget = 1;
} else if (g_str_has_prefix(verb, "POST")) {
curlopt_post = 1;
} else if (g_str_has_prefix(verb, "HEAD")) {
curlopt_nobody = 1;
} else {
curlopt_customrequest = verb;
}
if (size_func) {
request_body_size = size_func(read_data);
}
if (hdl->s3_api == S3_API_AWS4) {
if (read_data) {
data_SHA256Hash = s3_compute_sha256_hash_ba(read_data);
} else {
data_SHA256Hash = s3_compute_sha256_hash((unsigned char *)"", 0);
}
} else if (md5_func) {
md5_hash = md5_func(read_data);
if (md5_hash) {
md5_hash_b64 = s3_base64_encode(md5_hash);
md5_hash_hex = s3_hex_encode(md5_hash);
g_byte_array_free(md5_hash, TRUE);
}
}
if (!read_func) {
/* Curl will use fread() otherwise */
read_func = s3_empty_read_func;
}
if (write_func) {
int_writedata.write_func = write_func;
int_writedata.reset_func = write_reset_func;
int_writedata.write_data = write_data;
} else {
/* Curl will use fwrite() otherwise */
int_writedata.write_func = s3_counter_write_func;
int_writedata.reset_func = s3_counter_reset_func;
int_writedata.write_data = NULL;
}
while (1) {
struct curl_slist *header;
/* reset things */
if (headers) {
curl_slist_free_all(headers);
}
curl_error_buffer[0] = '\0';
if (read_reset_func) {
read_reset_func(read_data);
}
/* calls write_reset_func */
s3_internal_reset_func(&int_writedata);
/* set up the request */
headers = authenticate_request(hdl, verb, bucket, key, subresource, query,
md5_hash_b64, data_SHA256Hash, content_type, request_body_size, project_id);
/* add user header to headers */
for (header = user_headers; header != NULL; header = header->next) {
headers = curl_slist_append(headers, header->data);
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ))) {
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOSIGNAL, TRUE)))
goto curl_error;
if (hdl->ca_info) {
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CAINFO, hdl->ca_info)))
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_VERBOSE, hdl->verbose)))
goto curl_error;
if (hdl->verbose) {
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_DEBUGFUNCTION,
curl_debug_message)))
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_ERRORBUFFER,
curl_error_buffer)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS, 1)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_FOLLOWLOCATION, 1)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_URL, url)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPHEADER,
headers)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEFUNCTION, s3_internal_write_func)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEDATA, &int_writedata)))
goto curl_error;
/* Note: we always have to set this apparently, for consistent "end of header" detection */
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HEADERFUNCTION, s3_internal_header_func)))
goto curl_error;
/* Note: if set, CURLOPT_HEADERDATA seems to also be used for CURLOPT_WRITEDATA ? */
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HEADERDATA, &int_writedata)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_PROGRESSFUNCTION, progress_func)))
goto curl_error;
if (progress_func) {
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS,0)))
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_PROGRESSDATA, progress_data)))
goto curl_error;
if (!chunked) {
/* CURLOPT_INFILESIZE_LARGE added in 7.11.0 */
#if LIBCURL_VERSION_NUM >= 0x070b00
if ((curl_code = curl_easy_setopt(hdl->curl,
CURLOPT_INFILESIZE_LARGE,
(curl_off_t)request_body_size)))
goto curl_error;
#else
if ((curl_code = curl_easy_setopt(hdl->curl,
CURLOPT_INFILESIZE,
(long)request_body_size)))
goto curl_error;
#endif
/* CURLOPT_POSTFIELDSIZE_LARGE added in 7.11.1 */
#if LIBCURL_VERSION_NUM >= 0x070b01
if ((curl_code = curl_easy_setopt(hdl->curl,
CURLOPT_POSTFIELDSIZE_LARGE,
(curl_off_t)request_body_size)))
goto curl_error;
#else
if ((curl_code = curl_easy_setopt(hdl->curl,
CURLOPT_POSTFIELDSIZE,
(long)request_body_size)))
goto curl_error;
#endif
}
/* CURLOPT_MAX_{RECV,SEND}_SPEED_LARGE added in 7.15.5 */
#if LIBCURL_VERSION_NUM >= 0x070f05
if (s3_curl_throttling_compat()) {
if (hdl->max_send_speed)
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAX_SEND_SPEED_LARGE, (curl_off_t)hdl->max_send_speed)))
goto curl_error;
if (hdl->max_recv_speed)
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)hdl->max_recv_speed)))
goto curl_error;
}
#endif
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPGET, curlopt_httpget)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, curlopt_upload)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_POST, curlopt_post)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOBODY, curlopt_nobody)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CUSTOMREQUEST,
curlopt_customrequest)))
goto curl_error;
if (curlopt_upload || curlopt_post) {
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION, read_func)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, read_data)))
goto curl_error;
} else {
/* Clear request_body options. */
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION,
NULL)))
goto curl_error;
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA,
NULL)))
goto curl_error;
}
if (hdl->proxy) {
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_PROXY,
hdl->proxy)))
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_FRESH_CONNECT,
(long)(hdl->reuse_connection && retry_after_close == 0 ? 0 : 1)))) {
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_FORBID_REUSE,
(long)(hdl->reuse_connection? 0 : 1)))) {
goto curl_error;
}
if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_TIMEOUT,
(long)hdl->timeout))) {
goto curl_error;
}
/* Perform the request */
curl_code = curl_easy_perform(hdl->curl);
/* interpret the response into hdl->last* */
curl_error: /* (label for short-circuiting the curl_easy_perform call) */
should_retry = interpret_response(hdl, curl_code, curl_error_buffer,
int_writedata.resp_buf.buffer, int_writedata.resp_buf.buffer_pos, int_writedata.etag, md5_hash_hex);
if (hdl->last_response_code == 503) {
s3_new_curl(hdl);
}
if (hdl->s3_api == S3_API_OAUTH2 &&
hdl->last_response_code == 401 &&
hdl->last_s3_error_code == S3_ERROR_AuthenticationRequired) {
should_retry = oauth2_get_access_token(hdl);
}
/* and, unless we know we need to retry, see what we're to do now */
if (!should_retry) {
result = lookup_result(result_handling, hdl->last_response_code,
hdl->last_s3_error_code, hdl->last_curl_code);
/* break out of the while(1) unless we're retrying */
if (result != S3_RESULT_RETRY)
break;
}
if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES &&
retry_after_close < 3 &&
hdl->last_s3_error_code == S3_ERROR_RequestTimeout) {
retries = -1;
retry_after_close++;
g_debug("Retry on a new connection");
}
if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES) {
/* we're out of retries, so annotate hdl->last_message appropriately and bail
* out. */
char *m = g_strdup_printf("Too many retries; last message was '%s'", hdl->last_message);
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = m;
result = S3_RESULT_FAIL;
break;
}
g_usleep(backoff);
retries++;
backoff *= EXPONENTIAL_BACKOFF_BASE;
}
if (result != S3_RESULT_OK) {
g_debug(_("%s %s failed with %d/%s"), verb, url,
hdl->last_response_code,
s3_error_name_from_code(hdl->last_s3_error_code));
}
cleanup:
g_free(url);
if (headers) curl_slist_free_all(headers);
g_free(md5_hash_b64);
g_free(md5_hash_hex);
g_free(data_SHA256Hash);
/* we don't deallocate the response body -- we keep it for later */
g_free(hdl->etag);
hdl->etag = int_writedata.etag;
hdl->last_response_body = int_writedata.resp_buf.buffer;
hdl->last_response_body_size = int_writedata.resp_buf.buffer_pos;
hdl->last_num_retries = retries;
return result;
}
static size_t
s3_internal_write_func(void *ptr, size_t size, size_t nmemb, void * stream)
{
S3InternalData *data = (S3InternalData *) stream;
size_t bytes_saved;
if (!data->headers_done)
return size*nmemb;
/* call write on internal buffer (if not full) */
if (data->int_write_done) {
bytes_saved = 0;
} else {
bytes_saved = s3_buffer_write_func(ptr, size, nmemb, &data->resp_buf);
if (!bytes_saved) {
data->int_write_done = TRUE;
}
}
/* call write on user buffer */
if (data->write_func) {
return data->write_func(ptr, size, nmemb, data->write_data);
} else {
return bytes_saved;
}
}
static void
s3_internal_reset_func(void * stream)
{
S3InternalData *data = (S3InternalData *) stream;
s3_buffer_reset_func(&data->resp_buf);
data->headers_done = FALSE;
data->int_write_done = FALSE;
amfree(data->etag);
if (data->reset_func) {
data->reset_func(data->write_data);
}
}
static size_t
s3_internal_header_func(void *ptr, size_t size, size_t nmemb, void * stream)
{
static const char *final_header = "\r\n";
time_t remote_time_in_sec,local_time;
char *header;
regmatch_t pmatch[2];
S3InternalData *data = (S3InternalData *) stream;
header = g_strndup((gchar *) ptr, (gsize) size*nmemb);
if (header[strlen(header)-1] == '\n')
header[strlen(header)-1] = '\0';
if (header[strlen(header)-1] == '\r')
header[strlen(header)-1] = '\0';
if (!s3_regexec_wrap(&etag_regex, header, 2, pmatch, 0)) {
g_free(data->etag);
data->etag = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&x_auth_token_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->x_auth_token);
data->hdl->x_auth_token = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&x_subject_token_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->x_auth_token);
data->hdl->x_auth_token = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&x_storage_url_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->x_storage_url);
data->hdl->x_storage_url = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&content_type_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->content_type);
data->hdl->content_type = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&transfer_encoding_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->transfer_encoding);
data->hdl->transfer_encoding = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&x_amz_expiration_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->x_amz_expiration);
data->hdl->x_amz_expiration = find_regex_substring(header, pmatch[1]);
}
if (!s3_regexec_wrap(&x_amz_restore_regex, header, 2, pmatch, 0)) {
g_free(data->hdl->x_amz_restore);
data->hdl->x_amz_restore = find_regex_substring(header, pmatch[1]);
}
if (strlen(header) == 0)
data->headers_done = TRUE;
if (g_str_equal(final_header, header))
data->headers_done = TRUE;
if (g_str_equal("\n", header))
data->headers_done = TRUE;
/* If date header is found */
if (!s3_regexec_wrap(&date_sync_regex, header, 2, pmatch, 0)){
char *date = find_regex_substring(header, pmatch[1]);
/* Remote time is always in GMT: RFC 2616 */
/* both curl_getdate and time operate in UTC, so no timezone math is necessary */
if ( (remote_time_in_sec = curl_getdate(date, NULL)) < 0 ){
g_debug("Error: Conversion of remote time to seconds failed.");
data->hdl->time_offset_with_s3 = 0;
}else{
local_time = time(NULL);
/* Offset time */
data->hdl->time_offset_with_s3 = remote_time_in_sec - local_time;
if (data->hdl->verbose)
g_debug("Time Offset (remote - local) :%ld",(long)data->hdl->time_offset_with_s3);
}
g_free(date);
}
g_free(header);
return size*nmemb;
}
static gboolean
compile_regexes(void)
{
#ifdef HAVE_REGEX_H
/* using POSIX regular expressions */
struct {const char * str; int flags; regex_t *regex;} regexes[] = {
{"<Code>[[:space:]]*([^<]*)[[:space:]]*</Code>", REG_EXTENDED | REG_ICASE, &error_name_regex},
{"^ETag:[[:space:]]*\"([^\"]+)\"[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &etag_regex},
{"^X-Auth-Token:[[:space:]]*([^ ]+)[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &x_auth_token_regex},
{"^X-Subject-Token:[[:space:]]*([^ ]+)[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &x_subject_token_regex},
{"^X-Storage-Url:[[:space:]]*([^ ]+)[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &x_storage_url_regex},
{"^Content-Type:[[:space:]]*([^ ;]+).*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &content_type_regex},
{"^Transfer-Encoding:[[:space:]]*([^ ;]+).*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &transfer_encoding_regex},
{"<Message>[[:space:]]*([^<]*)[[:space:]]*</Message>", REG_EXTENDED | REG_ICASE, &message_regex},
{"^[a-z0-9](-*[a-z0-9]){2,62}$", REG_EXTENDED | REG_NOSUB, &subdomain_regex},
{"(/>)|(>([^<]*)</LocationConstraint>)", REG_EXTENDED | REG_ICASE, &location_con_regex},
{"^Date:(.*)$",REG_EXTENDED | REG_ICASE | REG_NEWLINE, &date_sync_regex},
{"\"access_token\" : \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &access_token_regex},
{"\"expires_in\" : (.*)", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &expires_in_regex},
{"\"details\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &details_regex},
{"\"code\": (.*),", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &code_regex},
{"\"message\": \"([^\"]*)\",", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &json_message_regex},
{"<UploadId>[[:space:]]*([^<]*)[[:space:]]*</UploadId>", REG_EXTENDED | REG_ICASE, &uploadId_regex},
{"<h1>[[:space:]]*([^<]*)[[:space:]]*</h1>", REG_EXTENDED | REG_ICASE, &html_error_name_regex},
{"<p>[[:space:]]*([^<]*)[[:space:]]*</p>", REG_EXTENDED | REG_ICASE, &html_message_regex},
{"^x-amz-expiration:[[:space:]]*([^ ]+)[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &x_amz_expiration_regex},
{"^x-amz-restore:[[:space:]]*([^ ]+)[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &x_amz_restore_regex},
{NULL, 0, NULL}
};
char regmessage[1024];
int i;
int reg_result;
for (i = 0; regexes[i].str; i++) {
reg_result = regcomp(regexes[i].regex, regexes[i].str, regexes[i].flags);
if (reg_result != 0) {
regerror(reg_result, regexes[i].regex, regmessage, sizeof(regmessage));
g_error(_("Regex error: %s"), regmessage);
return FALSE;
}
}
#else /* ! HAVE_REGEX_H */
/* using PCRE via GLib */
struct {const char * str; int flags; regex_t *regex;} regexes[] = {
{"<Code>\\s*([^<]*)\\s*</Code>",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&error_name_regex},
{"^ETag:\\s*\"([^\"]+)\"\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&etag_regex},
{"^X-Auth-Token:\\s*([^ ]+)\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&x_auth_token_regex},
{"^X-Subject-Token:\\s*([^ ]+)\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&x_subject_token_regex},
{"^X-Storage-Url:\\s*([^ ]+)\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&x_storage_url_regex},
{"^Content-Type:\\s*([^ ]+)\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&content_type_regex},
{"^Transfer-Encoding:\\s*([^ ]+)\\s*$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&transfer_encoding_regex},
{"<Message>\\s*([^<]*)\\s*</Message>",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&message_regex},
{"^[a-z0-9]((-*[a-z0-9])|(\\.[a-z0-9])){2,62}$",
G_REGEX_OPTIMIZE | G_REGEX_NO_AUTO_CAPTURE,
&subdomain_regex},
{"(/>)|(>([^<]*)</LocationConstraint>)",
G_REGEX_CASELESS,
&location_con_regex},
{"^Date:(.*)$",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&date_sync_regex},
{"\"access_token\" : \"([^\"]*)\"",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&access_token_regex},
{"\"expires_n\" : (.*)",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&expires_in_regex},
{"\"details\" : \"([^\"]*)\"",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&details_regex},
{"\"code\" : (.*)",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&code_regex},
{"\"message\" : \"([^\"]*)\"",
G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
&json_message_regex},
{"(/>)|(>([^<]*)</UploadId>)",
G_REGEX_CASELESS,
&uploadId_regex},
{"(/>)|(<h1>([^<]*)</h1>)",
G_REGEX_CASELESS,
&html_error_name_regex},
{"(/>)|(<p>([^<]*)</p>)",
G_REGEX_CASELESS,
&html_message_regex},
{NULL, 0, NULL}
};
int i;
GError *err = NULL;
for (i = 0; regexes[i].str; i++) {
*(regexes[i].regex) = g_regex_new(regexes[i].str, regexes[i].flags, 0, &err);
if (err) {
g_error(_("Regex error: %s"), err->message);
g_error_free(err);
return FALSE;
}
}
#endif
return TRUE;
}
/*
* Public function implementations
*/
#if (GLIB_MAJOR_VERSION > 2 || (GLIB_MAJOR_VERSION == 2 && GLIB_MINOR_VERSION >= 31))
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wmissing-field-initializers"
#endif
gboolean s3_init(void)
{
static GStaticMutex mutex = G_STATIC_MUTEX_INIT;
static gboolean init = FALSE, ret;
/* n.b. curl_global_init is called in common-src/glib-util.c:glib_init() */
g_static_mutex_lock (&mutex);
if (!init) {
ret = compile_regexes();
init = TRUE;
}
g_static_mutex_unlock(&mutex);
return ret;
}
#if (GLIB_MAJOR_VERSION > 2 || (GLIB_MAJOR_VERSION == 2 && GLIB_MINOR_VERSION >= 31))
# pragma GCC diagnostic pop
#endif
gboolean
s3_curl_location_compat(void)
{
curl_version_info_data *info;
info = curl_version_info(CURLVERSION_NOW);
return info->version_num > 0x070a02;
}
gboolean
s3_bucket_location_compat(const char *bucket)
{
return !s3_regexec_wrap(&subdomain_regex, bucket, 0, NULL, 0);
}
static gboolean
get_openstack_swift_api_v1_setting(
S3Handle *hdl)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, S3_ERROR_RegionNotFound, 0, S3_RESULT_FAIL },
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
s3_verbose(hdl, 1);
result = perform_request(hdl, "GET", NULL, NULL, NULL, NULL, NULL, NULL,
NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
return result == S3_RESULT_OK;
}
static gboolean
get_openstack_swift_api_v2_setting(
S3Handle *hdl)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
CurlBuffer buf = {NULL, 0, 0, 0, TRUE, NULL, NULL};
GString *body = g_string_new("");
g_string_append_printf(body, "{ \"auth\": {\n");
if (hdl->tenant_id) {
g_string_append_printf(body, "\"tenantId\":\"%s\"", hdl->tenant_id);
} else if (hdl->tenant_name) {
g_string_append_printf(body, "\"tenantName\":\"%s\"", hdl->tenant_name);
}
if (hdl->username && hdl->password) {
g_string_append_printf(body, ",\"passwordCredentials\": { \"username\":\"%s\", \"password\":\"%s\" }", hdl->username, hdl->password);
} else {
g_string_append_printf(body, ",\"apiAccessKeyCredentialsi\":{ \"accessKey\":\"%s\", \"secretKey\":\"%s\" }", hdl->access_key, hdl->secret_key);
}
g_string_append(body, "}}");
buf.buffer = g_string_free(body, FALSE);
buf.buffer_len = strlen(buf.buffer);
s3_verbose(hdl, 1);
hdl->getting_swift_2_token = TRUE;
g_free(hdl->x_auth_token);
hdl->x_auth_token = NULL;
g_free(hdl->x_storage_url);
hdl->x_storage_url = NULL;
result = perform_request(hdl, "POST", NULL, NULL, NULL, NULL,
"application/json", NULL, NULL,
S3_BUFFER_READ_FUNCS, &buf,
NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
hdl->getting_swift_2_token = FALSE;
return result == S3_RESULT_OK;
}
static gboolean
get_openstack_swift_api_v3_setting(
S3Handle *hdl)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 201, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
CurlBuffer buf = {NULL, 0, 0, 0, TRUE, NULL, NULL};
GString *body = g_string_new("");
g_string_append_printf(body, "{ \"auth\": {\n");
g_string_append_printf(body, " \"scope\": {\n");
g_string_append_printf(body, " \"project\": {\n");
g_string_append_printf(body, " \"domain\": {\n");
g_string_append_printf(body, " \"name\": \"%s\" },\n", hdl->domain_name);
g_string_append_printf(body, " \"name\": \"%s\" }},\n", hdl->project_name);
g_string_append_printf(body, " \"identity\": {\n");
g_string_append_printf(body, " \"methods\": [ \"password\" ],\n");
g_string_append_printf(body, " \"password\": {\n");
g_string_append_printf(body, " \"user\": {\n");
g_string_append_printf(body, " \"name\": \"%s\",\n", hdl->username);
g_string_append_printf(body, " \"domain\": {\n");
g_string_append_printf(body, " \"name\": \"%s\" },\n", hdl->domain_name);
g_string_append_printf(body, " \"password\": \"%s\" }}}}}\n", hdl->password);
buf.buffer = g_string_free(body, FALSE);
// g_debug("buffer: %s", buf.buffer);
buf.buffer_len = strlen(buf.buffer);
s3_verbose(hdl, 1);
hdl->getting_swift_3_token = TRUE;
g_free(hdl->x_auth_token);
hdl->x_auth_token = NULL;
g_free(hdl->x_storage_url);
hdl->x_storage_url = NULL;
result = perform_request(hdl, "POST", NULL, NULL, NULL, NULL,
"application/json", NULL, NULL,
S3_BUFFER_READ_FUNCS, &buf,
NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
hdl->getting_swift_3_token = FALSE;
return result == S3_RESULT_OK;
}
S3Handle *
s3_open(const char *access_key,
const char *secret_key,
const char *session_token,
const char *swift_account_id,
const char *swift_access_key,
const char *host,
const char *service_path,
const gboolean use_subdomain,
const char *user_token,
const char *bucket_location,
const char *storage_class,
const char *ca_info,
const char *server_side_encryption,
const char *proxy,
const S3_api s3_api,
const char *username,
const char *password,
const char *tenant_id,
const char *tenant_name,
const char *project_name,
const char *domain_name,
const char *client_id,
const char *client_secret,
const char *refresh_token,
const gboolean reuse_connection,
const gboolean read_from_glacier,
const long timeout,
const char *reps,
const char *reps_bucket)
{
S3Handle *hdl;
char *hwp;
hdl = g_new0(S3Handle, 1);
if (!hdl) goto error;
hdl->verbose = TRUE;
hdl->use_ssl = s3_curl_supports_ssl();
hdl->reuse_connection = reuse_connection;
hdl->read_from_glacier = read_from_glacier;
hdl->timeout = timeout;
if (s3_api == S3_API_S3) {
g_assert(access_key);
hdl->access_key = g_strdup(access_key);
g_assert(secret_key);
hdl->secret_key = g_strdup(secret_key);
/* NULL is okay */
hdl->session_token = g_strdup(session_token);
} else if (s3_api == S3_API_AWS4) {
g_assert(access_key);
hdl->access_key = g_strdup(access_key);
g_assert(secret_key);
hdl->secret_key = g_strdup(secret_key);
/* NULL is okay */
hdl->session_token = g_strdup(session_token);
} else if (s3_api == S3_API_SWIFT_1) {
g_assert(swift_account_id);
hdl->swift_account_id = g_strdup(swift_account_id);
g_assert(swift_access_key);
hdl->swift_access_key = g_strdup(swift_access_key);
} else if (s3_api == S3_API_SWIFT_2) {
g_assert((username && password) || (access_key && secret_key));
hdl->username = g_strdup(username);
hdl->password = g_strdup(password);
hdl->access_key = g_strdup(access_key);
hdl->secret_key = g_strdup(secret_key);
g_assert(tenant_id || tenant_name);
hdl->tenant_id = g_strdup(tenant_id);
hdl->tenant_name = g_strdup(tenant_name);
} else if (s3_api == S3_API_SWIFT_3) {
g_assert((username && password) || (access_key && secret_key));
hdl->username = g_strdup(username);
hdl->password = g_strdup(password);
hdl->access_key = g_strdup(access_key);
hdl->secret_key = g_strdup(secret_key);
hdl->tenant_id = g_strdup(tenant_id);
hdl->tenant_name = g_strdup(tenant_name);
if (project_name) {
hdl->project_name = g_strdup(project_name);
} else {
hdl->project_name = g_strdup(username);
}
if (domain_name) {
hdl->domain_name = g_strdup(domain_name);
} else {
hdl->domain_name = g_strdup("Default");
}
} else if (s3_api == S3_API_OAUTH2) {
hdl->client_id = g_strdup(client_id);
hdl->client_secret = g_strdup(client_secret);
hdl->refresh_token = g_strdup(refresh_token);
} else if (s3_api == S3_API_CASTOR) {
hdl->username = g_strdup(username);
hdl->password = g_strdup(password);
hdl->tenant_name = g_strdup(tenant_name);
hdl->reps = g_strdup(reps);
hdl->reps_bucket = g_strdup(reps_bucket);
}
/* NULL is okay */
hdl->user_token = g_strdup(user_token);
/* NULL is okay */
hdl->bucket_location = g_strdup(bucket_location);
/* NULL is ok */
hdl->storage_class = g_strdup(storage_class);
/* NULL is ok */
hdl->server_side_encryption = g_strdup(server_side_encryption);
/* NULL is ok */
hdl->proxy = g_strdup(proxy);
/* NULL is okay */
hdl->ca_info = g_strdup(ca_info);
if (!is_non_empty_string(host))
host = "s3.amazonaws.com";
hdl->host = g_ascii_strdown(host, -1);
hwp = strchr(hdl->host, ':');
if (hwp) {
*hwp = '\0';
hdl->host_without_port = g_strdup(hdl->host);
*hwp = ':';
} else {
hdl->host_without_port = g_strdup(hdl->host);
}
hdl->use_subdomain = use_subdomain ||
(g_str_equal(hdl->host, "s3.amazonaws.com") &&
is_non_empty_string(hdl->bucket_location));
hdl->s3_api = s3_api;
if (service_path) {
if (strlen(service_path) == 0 ||
(strlen(service_path) == 1 && service_path[0] == '/')) {
hdl->service_path = NULL;
} else if (service_path[0] != '/') {
hdl->service_path = g_strdup_printf("/%s", service_path);
} else {
hdl->service_path = g_strdup(service_path);
}
if (hdl->service_path) {
/* remove trailling / */
size_t len = strlen(hdl->service_path) - 1;
if (hdl->service_path[len] == '/')
hdl->service_path[len] = '\0';
}
} else {
hdl->service_path = NULL;
}
s3_new_curl(hdl);
if (!hdl->curl) goto error;
return hdl;
error:
s3_free(hdl);
return NULL;
}
static void
s3_new_curl(
S3Handle *hdl)
{
if (hdl->curl) {
curl_easy_cleanup(hdl->curl);
}
hdl->curl = curl_easy_init();
if (!hdl->curl) return;
/* Set HTTP handling options for CAStor */
if (hdl->s3_api == S3_API_CASTOR) {
#if LIBCURL_VERSION_NUM >= 0x071301
curl_version_info_data *info;
/* check the runtime version too */
info = curl_version_info(CURLVERSION_NOW);
if (info->version_num >= 0x071301) {
curl_easy_setopt(hdl->curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(hdl->curl, CURLOPT_UNRESTRICTED_AUTH, 1);
curl_easy_setopt(hdl->curl, CURLOPT_MAXREDIRS, 5);
curl_easy_setopt(hdl->curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
curl_easy_setopt(hdl->curl, CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_1_1);
if (hdl->username)
curl_easy_setopt(hdl->curl, CURLOPT_USERNAME, hdl->username);
if (hdl->password)
curl_easy_setopt(hdl->curl, CURLOPT_PASSWORD, hdl->password);
curl_easy_setopt(hdl->curl, CURLOPT_HTTPAUTH,
(CURLAUTH_BASIC | CURLAUTH_DIGEST));
}
#endif
}
}
gboolean
s3_open2(
S3Handle *hdl)
{
gboolean ret = TRUE;
/* get the X-Storage-Url and X-Auth-Token */
if (hdl->s3_api == S3_API_SWIFT_1) {
ret = get_openstack_swift_api_v1_setting(hdl);
} else if (hdl->s3_api == S3_API_SWIFT_2) {
ret = get_openstack_swift_api_v2_setting(hdl);
} else if (hdl->s3_api == S3_API_SWIFT_3) {
ret = get_openstack_swift_api_v3_setting(hdl);
}
return ret;
}
void
s3_free(S3Handle *hdl)
{
s3_reset(hdl);
if (hdl) {
g_free(hdl->access_key);
g_free(hdl->secret_key);
g_free(hdl->session_token);
g_free(hdl->swift_account_id);
g_free(hdl->swift_access_key);
g_free(hdl->content_type);
g_free(hdl->ca_info);
g_free(hdl->proxy);
g_free(hdl->username);
g_free(hdl->password);
g_free(hdl->tenant_id);
g_free(hdl->tenant_name);
g_free(hdl->project_name);
g_free(hdl->domain_name);
g_free(hdl->client_id);
g_free(hdl->client_secret);
g_free(hdl->refresh_token);
g_free(hdl->access_token);
g_free(hdl->etag);
g_free(hdl->transfer_encoding);
if (hdl->user_token) g_free(hdl->user_token);
if (hdl->bucket_location) g_free(hdl->bucket_location);
if (hdl->storage_class) g_free(hdl->storage_class);
if (hdl->server_side_encryption) g_free(hdl->server_side_encryption);
if (hdl->host) g_free(hdl->host);
if (hdl->service_path) g_free(hdl->service_path);
if (hdl->curl) curl_easy_cleanup(hdl->curl);
g_free(hdl);
}
}
void
s3_reset(S3Handle *hdl)
{
if (hdl) {
/* We don't call curl_easy_reset here, because doing that in curl
* < 7.16 blanks the default CA certificate path, and there's no way
* to get it back. */
if (hdl->last_message) {
g_free(hdl->last_message);
hdl->last_message = NULL;
}
hdl->last_response_code = 0;
hdl->last_curl_code = 0;
hdl->last_s3_error_code = 0;
hdl->last_num_retries = 0;
if (hdl->last_response_body) {
g_free(hdl->last_response_body);
hdl->last_response_body = NULL;
}
if (hdl->content_type) {
g_free(hdl->content_type);
hdl->content_type = NULL;
}
hdl->last_response_body_size = 0;
}
}
void
s3_error(S3Handle *hdl,
const char **message,
guint *response_code,
s3_error_code_t *s3_error_code,
const char **s3_error_name,
CURLcode *curl_code,
guint *num_retries)
{
if (hdl) {
if (message) *message = hdl->last_message;
if (response_code) *response_code = hdl->last_response_code;
if (s3_error_code) *s3_error_code = hdl->last_s3_error_code;
if (s3_error_name) *s3_error_name = s3_error_name_from_code(hdl->last_s3_error_code);
if (curl_code) *curl_code = hdl->last_curl_code;
if (num_retries) *num_retries = hdl->last_num_retries;
} else {
/* no hdl? return something coherent, anyway */
if (message) *message = "NULL S3Handle";
if (response_code) *response_code = 0;
if (s3_error_code) *s3_error_code = 0;
if (s3_error_name) *s3_error_name = NULL;
if (curl_code) *curl_code = 0;
if (num_retries) *num_retries = 0;
}
}
void
s3_verbose(S3Handle *hdl, gboolean verbose)
{
hdl->verbose = verbose;
}
gboolean
s3_set_max_send_speed(S3Handle *hdl, guint64 max_send_speed)
{
if (!s3_curl_throttling_compat())
return FALSE;
hdl->max_send_speed = max_send_speed;
return TRUE;
}
gboolean
s3_set_max_recv_speed(S3Handle *hdl, guint64 max_recv_speed)
{
if (!s3_curl_throttling_compat())
return FALSE;
hdl->max_recv_speed = max_recv_speed;
return TRUE;
}
gboolean
s3_use_ssl(S3Handle *hdl, gboolean use_ssl)
{
gboolean ret = TRUE;
if (use_ssl & !s3_curl_supports_ssl()) {
ret = FALSE;
} else {
hdl->use_ssl = use_ssl;
}
return ret;
}
char *
s3_strerror(S3Handle *hdl)
{
const char *message;
guint response_code;
const char *s3_error_name;
CURLcode curl_code;
guint num_retries;
char s3_info[256] = "";
char response_info[16] = "";
char curl_info[32] = "";
char retries_info[32] = "";
s3_error(hdl, &message, &response_code, NULL, &s3_error_name, &curl_code, &num_retries);
if (!message)
message = "Unknown S3 error";
if (s3_error_name)
g_snprintf(s3_info, sizeof(s3_info), " (%s)", s3_error_name);
if (response_code)
g_snprintf(response_info, sizeof(response_info), " (HTTP %d)", response_code);
if (curl_code)
g_snprintf(curl_info, sizeof(curl_info), " (CURLcode %d)", curl_code);
if (num_retries)
g_snprintf(retries_info, sizeof(retries_info), " (after %d retries)", num_retries);
return g_strdup_printf("%s%s%s%s%s", message, s3_info, curl_info, response_info, retries_info);
}
/* Perform an upload. When this function returns, KEY and
* BUFFER remain the responsibility of the caller.
*
* @param self: the s3 device
* @param bucket: the bucket to which the upload should be made
* @param key: the key to which the upload should be made
* @param buffer: the data to be uploaded
* @param buffer_len: the length of the data to upload
* @returns: false if an error ocurred
*/
gboolean
s3_upload(S3Handle *hdl,
const char *bucket,
const char *key,
const gboolean chunked,
s3_read_func read_func,
s3_reset_func reset_func,
s3_size_func size_func,
s3_md5_func md5_func,
gpointer read_data,
s3_progress_func progress_func,
gpointer progress_data)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 201, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
char *verb = "PUT";
char *content_type = NULL;
struct curl_slist *headers = NULL;
g_assert(hdl != NULL);
if (hdl->s3_api == S3_API_CASTOR) {
verb = "POST";
content_type = "application/x-amanda-backup-data";
}
if (chunked) {
headers = curl_slist_append(headers, "Transfer-Encoding: chunked");
size_func = NULL;
md5_func = NULL;
}
hdl->server_side_encryption_header = TRUE;
result = perform_request(hdl, verb, bucket, key, NULL,
NULL, content_type, NULL, headers,
read_func, reset_func, size_func, md5_func, read_data,
NULL, NULL, NULL, progress_func, progress_data,
result_handling, chunked);
hdl->server_side_encryption_header = FALSE;
return result == S3_RESULT_OK;
}
/* Perform an upload. When this function returns, KEY and
* BUFFER remain the responsibility of the caller.
*
* @param self: the s3 device
* @param bucket: the bucket to which the upload should be made
* @param key: the key to which the upload should be made
* @param buffer: the data to be uploaded
* @param buffer_len: the length of the data to upload
* @returns: false if an error ocurred
*/
gboolean
s3_part_upload(S3Handle *hdl,
const char *bucket,
const char *key,
const char *uploadId,
int partNumber,
char **etag,
s3_read_func read_func,
s3_reset_func reset_func,
s3_size_func size_func,
s3_md5_func md5_func,
gpointer read_data,
s3_progress_func progress_func,
gpointer progress_data)
{
char *subresource = NULL;
char **query = NULL;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
g_assert(hdl != NULL);
if (uploadId) {
if (hdl->s3_api == S3_API_AWS4) {
query = g_new0(char *, 3);
query[0] = g_strdup_printf("partNumber=%d", partNumber);
query[1] = g_strdup_printf("uploadId=%s", uploadId);
query[2] = NULL;
} else {
subresource = g_strdup_printf("partNumber=%d&uploadId=%s",
partNumber, uploadId);
}
}
result = perform_request(hdl, "PUT", bucket, key, subresource,
(const char **)query, NULL, NULL,
NULL,
read_func, reset_func, size_func, md5_func, read_data,
NULL, NULL, NULL, progress_func, progress_data,
result_handling, FALSE);
g_free(subresource);
if (query) {
g_free(query[0]);
g_free(query[1]);
g_free(query);
}
if (etag) {
*etag = hdl->etag;
hdl->etag = NULL;
}
return result == S3_RESULT_OK;
}
char *
s3_initiate_multi_part_upload(
S3Handle *hdl,
const char *bucket,
const char *key)
{
char *subresource;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
subresource = g_strdup_printf("uploads");
hdl->server_side_encryption_header = TRUE;
result = perform_request(hdl, "POST", bucket, key, subresource, NULL,
NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
hdl->server_side_encryption_header = FALSE;
g_free(subresource);
if (result == S3_RESULT_OK) {
return hdl->uploadId;
} else {
return NULL;
}
}
gboolean
s3_complete_multi_part_upload(
S3Handle *hdl,
const char *bucket,
const char *key,
const char *uploadId,
s3_read_func read_func,
s3_reset_func reset_func,
s3_size_func size_func,
s3_md5_func md5_func,
gpointer read_data)
{
char *subresource = NULL;
char **query = NULL;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
if (hdl->s3_api == S3_API_AWS4) {
query = g_new0(char *, 2);
query[0] = g_strdup_printf("uploadId=%s", uploadId);
query[1] = NULL;
} else {
subresource = g_strdup_printf("uploadId=%s", uploadId);
}
result = perform_request(hdl, "POST", bucket, key, subresource,
(const char **)query,
"application/xml", NULL, NULL,
read_func, reset_func, size_func, md5_func, read_data,
NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
g_free(subresource);
return (result == S3_RESULT_OK);
}
void
free_s3_object(
gpointer data)
{
s3_object *object = data;
g_free(object->key);
g_free(object->uploadId);
g_free(object->prefix);
g_free(object);
}
gboolean
s3_abort_multi_part_upload(
S3Handle *hdl,
const char *bucket,
const char *key,
const char *uploadId)
{
char *subresource = NULL;
char **query = NULL;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
if (hdl->s3_api == S3_API_AWS4) {
query = g_new0(char *, 2);
query[0] = g_strdup_printf("uploadId=%s", uploadId);
query[1] = NULL;
} else {
subresource = g_strdup_printf("uploadId=%s", uploadId);
}
result = perform_request(hdl, "DELETE", bucket, key, subresource,
(const char **)query,
"application/xml", NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
if (hdl->s3_api == S3_API_AWS4) {
g_free(query[0]);
g_free(query);
} else {
g_free(subresource);
}
return (result == S3_RESULT_OK);
}
/* Private structure for our "thunk", which tracks where the user is in the list
* of keys. */
struct list_keys_thunk {
GSList *object_list; /* all pending filenames */
s3_object *object;
gboolean in_contents; /* look for "key" entities in here */
gboolean in_common_prefixes; /* look for "prefix" entities in here */
gboolean is_truncated;
gchar *next_marker;
guint64 size;
gboolean want_text;
gchar *text;
gsize text_len;
};
/* Functions for a SAX parser to parse the XML from Amazon */
static void
list_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
const gchar **attribute_names G_GNUC_UNUSED,
const gchar **attribute_values G_GNUC_UNUSED,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
thunk->want_text = 0;
if (g_ascii_strcasecmp(element_name, "contents") == 0 ||
g_ascii_strcasecmp(element_name, "object") == 0 ||
g_ascii_strcasecmp(element_name, "upload") == 0) {
thunk->in_contents = 1;
thunk->object = g_new0(s3_object, 1);
} else if (g_ascii_strcasecmp(element_name, "commonprefixes") == 0) {
thunk->in_common_prefixes = 1;
thunk->object = g_new0(s3_object, 1);
} else if (g_ascii_strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
thunk->want_text = 1;
} else if ((g_ascii_strcasecmp(element_name, "key") == 0 ||
g_ascii_strcasecmp(element_name, "name") == 0) &&
thunk->in_contents) {
thunk->want_text = 1;
} else if ((g_ascii_strcasecmp(element_name, "size") == 0 ||
g_ascii_strcasecmp(element_name, "bytes") == 0) &&
thunk->in_contents) {
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "uploadid") == 0 && thunk->in_contents) {
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "istruncated")) {
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "nextmarker")) {
thunk->want_text = 1;
} else if (g_ascii_strcasecmp(element_name, "storageclass")) {
thunk->want_text = 1;
}
}
static void
list_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
if (g_ascii_strcasecmp(element_name, "contents") == 0 ||
g_ascii_strcasecmp(element_name, "object") == 0 ||
g_ascii_strcasecmp(element_name, "upload") == 0) {
thunk->in_contents = 0;
thunk->object_list = g_slist_prepend(thunk->object_list, thunk->object);
thunk->object = NULL;
} else if (g_ascii_strcasecmp(element_name, "commonprefixes") == 0) {
thunk->object_list = g_slist_prepend(thunk->object_list, thunk->object);
thunk->object = NULL;
thunk->in_common_prefixes = 0;
} else if ((g_ascii_strcasecmp(element_name, "key") == 0 ||
g_ascii_strcasecmp(element_name, "name") == 0) &&
thunk->in_contents) {
thunk->object->key = thunk->text;
if (thunk->is_truncated) {
if (thunk->next_marker) g_free(thunk->next_marker);
thunk->next_marker = g_strdup(thunk->text);
}
thunk->text = NULL;
} else if ((g_ascii_strcasecmp(element_name, "size") == 0 ||
g_ascii_strcasecmp(element_name, "bytes") == 0) &&
thunk->in_contents) {
thunk->object->size = g_ascii_strtoull (thunk->text, NULL, 10);
thunk->size += thunk->object->size;
g_free(thunk->text);
thunk->text = NULL;
} else if (g_ascii_strcasecmp(element_name, "uploadid") == 0 && thunk->in_contents) {
thunk->object->uploadId = thunk->text;
thunk->text = NULL;
} else if (g_ascii_strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
thunk->object->prefix = thunk->text;
thunk->text = NULL;
} else if (g_ascii_strcasecmp(element_name, "istruncated") == 0) {
if (thunk->text && g_ascii_strncasecmp(thunk->text, "false", 5) != 0)
thunk->is_truncated = TRUE;
} else if (g_ascii_strcasecmp(element_name, "nextmarker") == 0) {
if (thunk->next_marker) g_free(thunk->next_marker);
thunk->next_marker = thunk->text;
thunk->text = NULL;
} else if (g_ascii_strcasecmp(element_name, "storageclass") == 0) {
if (g_str_equal(thunk->text, "STANDARD")) {
thunk->object->storage_class = S3_SC_STANDARD;
} else if (g_str_equal(thunk->text, "STANDARD_IA")) {
thunk->object->storage_class = S3_SC_STANDARD_IA;
} else if (g_str_equal(thunk->text, "REDUCED_REDUNDANCY")) {
thunk->object->storage_class = S3_SC_REDUCED_REDUNDANCY;
} else if (g_str_equal(thunk->text, "GLACIER")) {
thunk->object->storage_class = S3_SC_GLACIER;
}
g_free(thunk->text);
thunk->text = NULL;
}
}
static void
list_text(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *text,
gsize text_len,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
if (thunk->want_text) {
if (thunk->text) g_free(thunk->text);
thunk->text = g_strndup(text, text_len);
}
}
/* Perform a fetch from S3; several fetches may be involved in a
* single listing operation */
static s3_result_t
list_fetch(S3Handle *hdl,
const char *bucket,
const char *subresource,
const char *prefix,
const char *delimiter,
const char *marker,
const char *max_keys,
CurlBuffer *buf)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
const char* pos_parts[][2] = {
{"delimiter", delimiter},
{"marker", marker},
{"max-keys", max_keys},
{"prefix", prefix},
{NULL, NULL}
};
char *esc_value;
char **query = g_new0(char *, 6);
char **q = query;
guint i;
/* loop over possible parts to build query string */
for (i = 0; pos_parts[i][0]; i++) {
if (pos_parts[i][1]) {
const char *keyword;
esc_value = curl_escape(pos_parts[i][1], 0);
keyword = pos_parts[i][0];
if ((hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3) &&
strcmp(keyword, "max-keys") == 0) {
keyword = "limit";
} else if ((hdl->s3_api == S3_API_CASTOR) &&
strcmp(keyword, "max-keys") == 0) {
keyword = "size";
}
*q++ = g_strdup_printf("%s=%s", keyword, esc_value);
curl_free(esc_value);
}
}
if (hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3 ||
hdl->s3_api == S3_API_CASTOR) {
*q++ = g_strdup("format=xml");
}
/* and perform the request on that URI */
result = perform_request(hdl, "GET", bucket, NULL, subresource,
(const char **)query, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL,
S3_BUFFER_WRITE_FUNCS, buf, NULL, NULL,
result_handling, FALSE);
for (q = query; *q != NULL; q++) {
g_free(*q);
}
return result;
}
gboolean
s3_list_keys(S3Handle *hdl,
const char *bucket,
const char *subresource,
const char *prefix,
const char *delimiter,
GSList **list,
guint64 *total_size)
{
/*
* max len of XML variables:
* bucket: 255 bytes (p12 API Version 2006-03-01)
* key: 1024 bytes (p15 API Version 2006-03-01)
* size per key: 5GB bytes (p6 API Version 2006-03-01)
* size of size 10 bytes (i.e. 10 decimal digits)
* etag: 44 (observed+assumed)
* owner ID: 64 (observed+assumed)
* owner DisplayName: 255 (assumed)
* StorageClass: STANDARD | REDUCED_REDUNDANCY | GLACIER
*/
static const guint MAX_RESPONSE_LEN = 1000*2000;
static const char *MAX_KEYS = "1000";
struct list_keys_thunk thunk;
GMarkupParseContext *ctxt = NULL;
static GMarkupParser parser = { list_start_element, list_end_element, list_text, NULL, NULL };
GError *err = NULL;
s3_result_t result = S3_RESULT_FAIL;
CurlBuffer buf = {NULL, 0, 0, MAX_RESPONSE_LEN, TRUE, NULL, NULL};
g_assert(list);
*list = NULL;
thunk.object_list = NULL;
thunk.object = NULL;
thunk.text = NULL;
thunk.next_marker = NULL;
thunk.size = 0;
/* Loop until S3 has given us the entire picture */
do {
s3_buffer_reset_func(&buf);
/* get some data from S3 */
result = list_fetch(hdl, bucket, subresource, prefix, delimiter, thunk.next_marker, MAX_KEYS, &buf);
if (result != S3_RESULT_OK) goto cleanup;
if (buf.buffer_pos == 0) goto cleanup; /* no body */
/* run the parser over it */
thunk.in_contents = FALSE;
thunk.in_common_prefixes = FALSE;
thunk.is_truncated = FALSE;
if (thunk.next_marker) g_free(thunk.next_marker);
thunk.next_marker = NULL;
thunk.want_text = FALSE;
ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
if (!g_markup_parse_context_parse(ctxt, buf.buffer, buf.buffer_pos, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
result = S3_RESULT_FAIL;
goto cleanup;
}
if (!g_markup_parse_context_end_parse(ctxt, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
result = S3_RESULT_FAIL;
goto cleanup;
}
g_markup_parse_context_free(ctxt);
ctxt = NULL;
} while (thunk.next_marker);
cleanup:
if (err) g_error_free(err);
if (thunk.text) g_free(thunk.text);
if (thunk.next_marker) g_free(thunk.next_marker);
if (ctxt) g_markup_parse_context_free(ctxt);
if (buf.buffer) g_free(buf.buffer);
if (result != S3_RESULT_OK) {
slist_free_full(thunk.object_list, free_s3_object);
return FALSE;
} else {
*list = thunk.object_list;
if(total_size) {
*total_size = thunk.size;
}
return TRUE;
}
}
gboolean
s3_init_restore(
S3Handle *hdl,
const char *bucket,
const char *key)
{
CurlBuffer data;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 202, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
data.buffer = "<RestoreRequest xmlns=\"http://s3.amazonaws.com/doc/2006-3-01\"> <Days>4</Days> </RestoreRequest>";
data.buffer_len = strlen(data.buffer);
data.buffer_pos = 0;
data.max_buffer_size = data.buffer_len;
data.end_of_buffer = TRUE;
data.mutex = NULL;
data.cond = NULL;
result = perform_request(hdl, "POST", bucket, key, "restore", NULL,
"application/xml", NULL, NULL,
s3_buffer_read_func, s3_buffer_reset_func,
s3_buffer_size_func, s3_buffer_md5_func,
&data,
NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
return (result == S3_RESULT_OK);
}
s3_head_t *
s3_head(
S3Handle *hdl,
const char *bucket,
const char *key)
{
s3_result_t result = S3_RESULT_FAIL;
s3_head_t *head;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
amfree(hdl->x_amz_expiration);
amfree(hdl->x_amz_restore);
result = perform_request(hdl, "HEAD", bucket, key, NULL, NULL, NULL, NULL,
NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
if (result != S3_RESULT_OK) {
return NULL;
}
head = g_new0(s3_head_t, 1);
head->key = g_strdup(key);
head->x_amz_expiration = g_strdup(hdl->x_amz_expiration);
head->x_amz_restore = g_strdup(hdl->x_amz_restore);
return head;
}
void
free_s3_head(
s3_head_t *head)
{
g_free(head->key);
g_free(head->x_amz_expiration);
g_free(head->x_amz_restore);
g_free(head);
}
gboolean
s3_read(S3Handle *hdl,
const char *bucket,
const char *key,
s3_write_func write_func,
s3_reset_func reset_func,
gpointer write_data,
s3_progress_func progress_func,
gpointer progress_data)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
int timeout = 300; /* 5 minutes */
g_assert(hdl != NULL);
g_assert(write_func != NULL);
while (1) {
result = perform_request(hdl, "GET", bucket, key, NULL, NULL, NULL,
NULL, NULL,
NULL, NULL, NULL, NULL, NULL, write_func, reset_func, write_data,
progress_func, progress_data, result_handling, FALSE);
if (!hdl->read_from_glacier ||
result != S3_RESULT_FAIL ||
hdl->last_response_code != 403 ||
hdl->last_s3_error_code != S3_ERROR_InvalidObjectState) {
break;
}
/* retry if a restore from glacier is ongoing */
sleep(timeout);
}
return result == S3_RESULT_OK;
}
gboolean
s3_read_range(S3Handle *hdl,
const char *bucket,
const char *key,
const guint64 range_begin,
const guint64 range_end,
s3_write_func write_func,
s3_reset_func reset_func,
gpointer write_data,
s3_progress_func progress_func,
gpointer progress_data)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 206, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
struct curl_slist *headers = NULL;
char *buf;
int timeout = 300; /* 5 minutes */
g_assert(hdl != NULL);
g_assert(write_func != NULL);
buf = g_strdup_printf("Range: bytes=%llu-%llu",
(long long unsigned)range_begin,
(long long unsigned) range_end);
headers = curl_slist_append(headers, buf);
g_free(buf);
while(1) {
result = perform_request(hdl, "GET", bucket, key, NULL, NULL, NULL,
NULL, headers,
NULL, NULL, NULL, NULL, NULL, write_func, reset_func, write_data,
progress_func, progress_data, result_handling, FALSE);
if (!hdl->read_from_glacier ||
result != S3_RESULT_FAIL ||
hdl->last_response_code != 403 ||
hdl->last_s3_error_code != S3_ERROR_InvalidObjectState) {
break;
}
/* retry if a restore from glacier is ongoing */
sleep(timeout);
}
curl_slist_free_all(headers);
return result == S3_RESULT_OK;
}
gboolean
s3_delete(S3Handle *hdl,
const char *bucket,
const char *key)
{
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
{ 404, 0, 0, S3_RESULT_OK },
{ 404, S3_ERROR_NoSuchBucket, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 409, 0, 0, S3_RESULT_OK },
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
g_assert(hdl != NULL);
result = perform_request(hdl, "DELETE", bucket, key, NULL, NULL,
"application/xml", NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
return result == S3_RESULT_OK;
}
int
s3_multi_delete(S3Handle *hdl,
const char *bucket,
GSList *objects)
{
GString *query;
CurlBuffer data;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
{ 400, 0, 0, S3_RESULT_NOTIMPL },
{ 403, 0, 0, S3_RESULT_NOTIMPL },
{ 404, S3_ERROR_NoSuchBucket, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
g_assert(hdl != NULL);
query = g_string_new(NULL);
if (hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3) {
char *container = s3_uri_encode(bucket, TRUE);
char *cmd = "DELETE";
while (objects != NULL) {
s3_object *object = objects->data;
char *name = s3_uri_encode(object->key, TRUE);
g_string_append_printf(query, "%s/%s\n", container, name);
objects = objects->next;
}
data.buffer_len = query->len;
data.buffer = query->str;
data.buffer_pos = 0;
data.max_buffer_size = data.buffer_len;
data.end_of_buffer = TRUE;
data.mutex = NULL;
data.cond = NULL;
if (hdl->s3_api == S3_API_SWIFT_3)
cmd = "POST";
result = perform_request(hdl, cmd, "", NULL, "bulk-delete", NULL,
"text/plain", NULL, NULL,
s3_buffer_read_func, s3_buffer_reset_func,
s3_buffer_size_func, s3_buffer_md5_func,
&data, NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
} else {
g_string_append(query, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
g_string_append(query, "<Delete>\n");
if (!hdl->verbose) {
g_string_append(query, " <Quiet>true</Quiet>\n");
}
while (objects != NULL) {
s3_object *object = objects->data;
g_string_append(query, " <Object>\n");
g_string_append(query, " <Key>");
g_string_append(query, object->key);
g_string_append(query, "</Key>\n");
g_string_append(query, " </Object>\n");
objects = objects->next;
}
g_string_append(query, "</Delete>\n");
data.buffer_len = query->len;
data.buffer = query->str;
data.buffer_pos = 0;
data.max_buffer_size = data.buffer_len;
data.end_of_buffer = TRUE;
data.mutex = NULL;
data.cond = NULL;
result = perform_request(hdl, "POST", bucket, NULL, "delete", NULL,
"application/xml", NULL, NULL,
s3_buffer_read_func, s3_buffer_reset_func,
s3_buffer_size_func, s3_buffer_md5_func,
&data, NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
}
g_string_free(query, TRUE);
if (result == S3_RESULT_OK) {
return 1;
} else if (result == S3_RESULT_NOTIMPL) {
s3_new_curl(hdl);
return 2;
} else {
return 0;
}
}
gboolean
s3_make_bucket(S3Handle *hdl,
const char *bucket,
const char *project_id)
{
char *body = NULL;
char *verb = "PUT";
char *content_type = NULL;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 201, 0, 0, S3_RESULT_OK },
{ 202, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
{ 404, S3_ERROR_NoSuchBucket, 0, S3_RESULT_RETRY },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
regmatch_t pmatch[4];
CurlBuffer buf = {NULL, 0, 0, 0, TRUE, NULL, NULL}, *ptr = NULL;
s3_read_func read_func = NULL;
s3_reset_func reset_func = NULL;
s3_md5_func md5_func = NULL;
s3_size_func size_func = NULL;
GString *CreateBucketConfiguration;
gboolean add_create = FALSE;
g_assert(hdl != NULL);
CreateBucketConfiguration = g_string_new("<CreateBucketConfiguration");
if (g_str_equal(hdl->host, "gss.iijgio.com")) {
g_string_append(CreateBucketConfiguration,
" xmlns=\"http://acs.iijgio.com/doc/2006-03-01/\"");
}
g_string_append(CreateBucketConfiguration, ">");
if (is_non_empty_string(hdl->bucket_location) &&
strcmp(hdl->bucket_location, "us-east-1") != 0 &&
!g_str_equal(AMAZON_WILDCARD_LOCATION, hdl->bucket_location)) {
if (s3_bucket_location_compat(bucket)) {
g_string_append_printf(CreateBucketConfiguration,
"<LocationConstraint>%s</LocationConstraint>",
hdl->bucket_location);
add_create = TRUE;
} else {
hdl->last_message = g_strdup_printf(_(
"Location constraint given for %s, "
"but the bucket name (%s) is not usable as a subdomain."), S3_bucket_name[hdl->s3_api], bucket);
g_string_free(CreateBucketConfiguration, TRUE);
return FALSE;
}
}
if (hdl->s3_api == S3_API_OAUTH2 && hdl->storage_class) {
g_string_append_printf(CreateBucketConfiguration,
"<StorageClass>%s</StorageClass>",
hdl->storage_class);
add_create = TRUE;
}
g_string_append(CreateBucketConfiguration, "</CreateBucketConfiguration>");
if (hdl->s3_api == S3_API_CASTOR) {
verb = "POST";
content_type = "application/castorcontext";
}
if (add_create) {
ptr = &buf;
buf.buffer = g_string_free(CreateBucketConfiguration, FALSE);
buf.buffer_len = (guint) strlen(buf.buffer);
buf.buffer_pos = 0;
buf.max_buffer_size = buf.buffer_len;
read_func = s3_buffer_read_func;
reset_func = s3_buffer_reset_func;
size_func = s3_buffer_size_func;
md5_func = s3_buffer_md5_func;
} else {
g_string_free(CreateBucketConfiguration, TRUE);
}
result = perform_request(hdl, verb, bucket, NULL, NULL,
NULL, content_type, project_id, NULL,
read_func, reset_func, size_func, md5_func, ptr,
NULL, NULL, NULL, NULL, NULL, result_handling, FALSE);
if (ptr)
g_free(ptr->buffer);
if (result == S3_RESULT_OK ||
(result != S3_RESULT_OK &&
hdl->last_s3_error_code == S3_ERROR_BucketAlreadyOwnedByYou)) {
/* verify the that the location constraint on the existing bucket matches
* the one that's configured.
*/
if (is_non_empty_string(hdl->bucket_location)) {
result = perform_request(hdl, "GET", bucket, NULL, "location", NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
} else {
result = perform_request(hdl, "GET", bucket, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
}
if (result == S3_RESULT_OK && is_non_empty_string(hdl->bucket_location)) {
/* return to the default state of failure */
result = S3_RESULT_FAIL;
/* use strndup to get a null-terminated string */
body = g_strndup(hdl->last_response_body, hdl->last_response_body_size);
if (!body) {
hdl->last_message = g_strdup(_("No body received for location request"));
goto cleanup;
} else if ('\0' == body[0]) {
hdl->last_message = g_strdup(_("Empty body received for location request"));
goto cleanup;
}
if (!s3_regexec_wrap(&location_con_regex, body, 4, pmatch, 0)) {
char *loc_end_open = find_regex_substring(body, pmatch[1]);
char *loc_content = find_regex_substring(body, pmatch[3]);
/* The case of an empty string is special because XML allows
* "self-closing" tags
*/
if (g_str_equal(AMAZON_WILDCARD_LOCATION,
hdl->bucket_location) &&
'/' != loc_end_open[0])
hdl->last_message = g_strdup(_("A wildcard location constraint is "
"configured, but the bucket has a non-empty location constraint"));
else if (!g_str_equal(AMAZON_WILDCARD_LOCATION,
hdl->bucket_location)?
strncmp(loc_content, hdl->bucket_location, strlen(hdl->bucket_location)) :
('\0' != loc_content[0]))
hdl->last_message = g_strdup_printf("The location constraint configured (%s) "
"does not match the constraint currently on the bucket (%s)", hdl->bucket_location, loc_content);
else
result = S3_RESULT_OK;
g_free(loc_end_open);
g_free(loc_content);
} else {
hdl->last_message = g_strdup_printf(_("Unexpected location response from %s"), S3_name[hdl->s3_api]);
}
}
}
cleanup:
if (body) g_free(body);
return result == S3_RESULT_OK;
}
static s3_result_t
oauth2_get_access_token(
S3Handle *hdl)
{
GString *query;
CurlBuffer data;
s3_result_t result = S3_RESULT_FAIL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
char *body;
regmatch_t pmatch[2];
g_assert(hdl != NULL);
query = g_string_new(NULL);
g_string_append(query, "client_id=");
g_string_append(query, hdl->client_id);
g_string_append(query, "&client_secret=");
g_string_append(query, hdl->client_secret);
g_string_append(query, "&refresh_token=");
g_string_append(query, hdl->refresh_token);
g_string_append(query, "&grant_type=refresh_token");
data.buffer_len = query->len;
data.buffer = query->str;
data.buffer_pos = 0;
data.max_buffer_size = data.buffer_len;
data.end_of_buffer = TRUE;
data.mutex = NULL;
data.cond = NULL;
hdl->x_storage_url = "https://accounts.google.com/o/oauth2/token";
hdl->getting_oauth2_access_token = 1;
result = perform_request(hdl, "POST", NULL, NULL, NULL, NULL,
"application/x-www-form-urlencoded", NULL, NULL,
s3_buffer_read_func, s3_buffer_reset_func,
s3_buffer_size_func, s3_buffer_md5_func,
&data, NULL, NULL, NULL, NULL, NULL,
result_handling, FALSE);
hdl->x_storage_url = NULL;
hdl->getting_oauth2_access_token = 0;
/* use strndup to get a null-terminated string */
body = g_strndup(hdl->last_response_body, hdl->last_response_body_size);
if (!body) {
hdl->last_message = g_strdup(_("No body received for location request"));
goto cleanup;
} else if ('\0' == body[0]) {
hdl->last_message = g_strdup(_("Empty body received for location request"));
goto cleanup;
}
if (!s3_regexec_wrap(&access_token_regex, body, 2, pmatch, 0)) {
hdl->access_token = find_regex_substring(body, pmatch[1]);
hdl->x_auth_token = g_strdup(hdl->access_token);
}
if (!s3_regexec_wrap(&expires_in_regex, body, 2, pmatch, 0)) {
char *expires_in = find_regex_substring(body, pmatch[1]);
hdl->expires = time(NULL) + atoi(expires_in) - 600;
g_free(expires_in);
}
cleanup:
g_free(body);
return result == S3_RESULT_OK;
}
gboolean
s3_is_bucket_exists(S3Handle *hdl,
const char *bucket,
const char *prefix,
const char *project_id)
{
s3_result_t result = S3_RESULT_FAIL;
char **query = g_new0(char *, 3);
char **q = query;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
{ 204, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
if (hdl->s3_api == S3_API_SWIFT_1 ||
hdl->s3_api == S3_API_SWIFT_2 ||
hdl->s3_api == S3_API_SWIFT_3) {
*q++ = g_strdup("limit=1");
} else if (hdl->s3_api == S3_API_CASTOR) {
*q++ = g_strdup("format=xml");
*q++ = g_strdup("size=0");
} else if (prefix) {
char *q_prefix = curl_escape(prefix, 0);
*q++ = g_strdup("max-keys=1");
*q++ = g_strdup_printf("prefix=%s", q_prefix);
g_free(q_prefix);
} else {
*q++ = g_strdup("max-keys=1");
}
result = perform_request(hdl, "GET", bucket, NULL, NULL,
(const char **)query,
NULL, project_id, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
for (q = query; *q != NULL; q++) {
g_free(*q);
}
return result == S3_RESULT_OK;
}
gboolean
s3_delete_bucket(S3Handle *hdl,
const char *bucket)
{
return s3_delete(hdl, bucket, NULL);
}
/* Private structure for our "thunk", which tracks where the user is in the list
* of keys. */
typedef struct lifecycle_thunk {
GSList *lifecycle; /* all rules */
lifecycle_rule *rule;
lifecycle_action *action;
gboolean in_LifecycleConfiguration;
gboolean in_Rule;
gboolean in_ID;
gboolean in_Prefix;
gboolean in_Status;
gboolean in_Transition;
gboolean in_Expiration;
gboolean in_Days;
gboolean in_Date;
gboolean in_StorageClass;
gboolean want_text;
gchar *text;
gsize text_len;
gchar *error;
} lifecycle_thunk;
void
free_lifecycle_rule(
gpointer data)
{
lifecycle_rule *rule = (lifecycle_rule *)data;
g_free(rule->id);
g_free(rule->prefix);
g_free(rule->status);
if (rule->transition) {
g_free(rule->transition->date);
g_free(rule->transition->storage_class);
g_free(rule->transition);
}
if (rule->expiration) {
g_free(rule->expiration->date);
g_free(rule->expiration->storage_class);
g_free(rule->expiration);
}
g_free(rule);
}
void
free_lifecycle(
GSList *lifecycle)
{
slist_free_full(lifecycle, free_lifecycle_rule);
}
/* Functions for a SAX parser to parse the XML from Amazon */
static void
lifecycle_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
const gchar **attribute_names G_GNUC_UNUSED,
const gchar **attribute_values G_GNUC_UNUSED,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
struct lifecycle_thunk *thunk = (lifecycle_thunk *)user_data;
thunk->want_text = FALSE;
if (g_ascii_strcasecmp(element_name, "lifecycleconfiguration") == 0) {
thunk->in_LifecycleConfiguration = TRUE;
} else if (g_ascii_strcasecmp(element_name, "rule") == 0) {
thunk->in_Rule = TRUE;
thunk->rule = g_new0(lifecycle_rule, 1);
} else if (g_ascii_strcasecmp(element_name, "id") == 0) {
thunk->in_ID = TRUE;
thunk->want_text = TRUE;
} else if (g_ascii_strcasecmp(element_name, "prefix") == 0) {
thunk->in_Prefix = TRUE;
thunk->want_text = TRUE;
} else if (g_ascii_strcasecmp(element_name, "status") == 0) {
thunk->in_Status = TRUE;
thunk->want_text = TRUE;
} else if (g_ascii_strcasecmp(element_name, "transition") == 0) {
thunk->in_Transition = TRUE;
thunk->action = g_new0(lifecycle_action, 1);
} else if (g_ascii_strcasecmp(element_name, "expiration") == 0) {
thunk->in_Expiration = TRUE;
thunk->action = g_new0(lifecycle_action, 1);
} else if (g_ascii_strcasecmp(element_name, "days") == 0) {
thunk->in_Days = TRUE;
thunk->want_text = TRUE;
} else if (g_ascii_strcasecmp(element_name, "date") == 0) {
thunk->in_Date = TRUE;
thunk->want_text = TRUE;
} else if (g_ascii_strcasecmp(element_name, "storageclass") == 0) {
thunk->in_StorageClass = TRUE;
thunk->want_text = TRUE;
} else {
g_free(thunk->error);
thunk->error = g_strdup("Unknown element name in lifecycle get");
}
}
static void
lifecycle_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *element_name,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
lifecycle_thunk *thunk = (lifecycle_thunk *)user_data;
if (g_ascii_strcasecmp(element_name, "lifecycleconfiguration") == 0) {
thunk->in_LifecycleConfiguration = FALSE;
} else if (g_ascii_strcasecmp(element_name, "rule") == 0) {
thunk->in_Rule = FALSE;
thunk->lifecycle = g_slist_prepend(thunk->lifecycle, thunk->rule);
thunk->rule = NULL;
} else if (g_ascii_strcasecmp(element_name, "id") == 0) {
thunk->in_ID = FALSE;
thunk->rule->id = thunk->text;
thunk->text = NULL;
thunk->want_text = FALSE;
} else if (g_ascii_strcasecmp(element_name, "prefix") == 0) {
thunk->in_Prefix = FALSE;
thunk->rule->prefix = thunk->text;
thunk->text = NULL;
thunk->want_text = FALSE;
} else if (g_ascii_strcasecmp(element_name, "status") == 0) {
thunk->in_Status = FALSE;
thunk->rule->status = thunk->text;
thunk->text = NULL;
thunk->want_text = FALSE;
} else if (g_ascii_strcasecmp(element_name, "transition") == 0) {
thunk->in_Transition = FALSE;
thunk->rule->transition = thunk->action;
thunk->action = NULL;
} else if (g_ascii_strcasecmp(element_name, "expiration") == 0) {
thunk->in_Expiration = FALSE;
thunk->rule->expiration = thunk->action;
thunk->action = NULL;
} else if (g_ascii_strcasecmp(element_name, "days") == 0) {
thunk->in_Days = FALSE;
thunk->action->days = atoi(thunk->text);
g_free(thunk->text);
thunk->text = NULL;
thunk->want_text = FALSE;
} else if (g_ascii_strcasecmp(element_name, "date") == 0) {
thunk->in_Date = FALSE;
thunk->action->date = thunk->text;
thunk->text = NULL;
thunk->want_text = FALSE;
} else if (g_ascii_strcasecmp(element_name, "storageclass") == 0) {
thunk->in_StorageClass = FALSE;
thunk->action->storage_class = thunk->text;
thunk->text = NULL;
thunk->want_text = FALSE;
}
}
static void
lifecycle_text(GMarkupParseContext *context G_GNUC_UNUSED,
const gchar *text,
gsize text_len,
gpointer user_data,
GError **error G_GNUC_UNUSED)
{
lifecycle_thunk *thunk = (lifecycle_thunk *)user_data;
if (thunk->want_text) {
if (thunk->text) g_free(thunk->text);
thunk->text = g_strndup(text, text_len);
}
}
gboolean
s3_get_lifecycle(
S3Handle *hdl,
const char *bucket,
GSList **lifecycle)
{
s3_result_t result = S3_RESULT_FAIL;
CurlBuffer buf = {NULL, 0, 0, 100000, TRUE, NULL, NULL};
struct lifecycle_thunk thunk;
GMarkupParseContext *ctxt = NULL;
static GMarkupParser parser = { lifecycle_start_element, lifecycle_end_element, lifecycle_text, NULL, NULL };
GError *err = NULL;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
thunk.lifecycle = NULL;
thunk.rule = NULL;
thunk.action = NULL;
thunk.in_LifecycleConfiguration = FALSE;
thunk.in_Rule = FALSE;
thunk.in_ID = FALSE;
thunk.in_Prefix = FALSE;
thunk.in_Status = FALSE;
thunk.in_Transition = FALSE;
thunk.in_Expiration = FALSE;
thunk.in_Days = FALSE;
thunk.in_Date = FALSE;
thunk.in_StorageClass = FALSE;
thunk.want_text = FALSE;
thunk.text = NULL;
thunk.text_len = 0;
thunk.error = NULL;
result = perform_request(hdl, "GET", bucket, NULL, "lifecycle", NULL,
NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
S3_BUFFER_WRITE_FUNCS, &buf,
NULL, NULL, result_handling, FALSE);
if (result == S3_RESULT_FAIL &&
hdl->last_response_code == 404 &&
hdl->last_s3_error_code == S3_ERROR_NoSuchLifecycleConfiguration) {
result = S3_RESULT_OK;
return TRUE;
}
if (result != S3_RESULT_OK) goto cleanup;
if (buf.buffer_pos == 0) goto cleanup;
/* run the parser over it */
ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
if (!g_markup_parse_context_parse(ctxt, buf.buffer, buf.buffer_pos, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
result = S3_RESULT_FAIL;
goto cleanup;
}
if (!g_markup_parse_context_end_parse(ctxt, &err)) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = g_strdup(err->message);
result = S3_RESULT_FAIL;
goto cleanup;
}
g_markup_parse_context_free(ctxt);
ctxt = NULL;
if (thunk.error) {
if (hdl->last_message) g_free(hdl->last_message);
hdl->last_message = thunk.error;
thunk.error = NULL;
result = S3_RESULT_FAIL;
goto cleanup;
}
cleanup:
if (err) g_error_free(err);
if (thunk.text) g_free(thunk.text);
if (ctxt) g_markup_parse_context_free(ctxt);
if (buf.buffer) g_free(buf.buffer);
if (result == S3_RESULT_OK) {
*lifecycle = thunk.lifecycle;
} else {
free_lifecycle(thunk.lifecycle);
}
return result == S3_RESULT_OK;
}
gboolean
s3_put_lifecycle(
S3Handle *hdl,
const char *bucket,
GSList *lifecycle)
{
s3_result_t result = S3_RESULT_FAIL;
CurlBuffer buf = {NULL, 0, 0, 0, TRUE, NULL, NULL};
GString *body = g_string_new("<LifecycleConfiguration>");
GSList *life;
lifecycle_rule *rule;
static result_handling_t result_handling[] = {
{ 200, 0, 0, S3_RESULT_OK },
RESULT_HANDLING_ALWAYS_RETRY,
{ 0, 0, 0, /* default: */ S3_RESULT_FAIL }
};
for (life = lifecycle; life != NULL; life = life->next) {
rule = (lifecycle_rule *)life->data;
g_string_append_printf(body,
"<Rule><ID>%s</ID><Prefix>%s</Prefix><Status>%s</Status>",
rule->id, rule->prefix, rule->status);
if (rule->transition) {
g_string_append(body, "<Transition>");
if (rule->transition->date) {
g_string_append_printf(body, "<Date>%s</Date>",
rule->transition->date);
} else {
g_string_append_printf(body, "<Days>%u</Days>",
rule->transition->days);
}
g_string_append_printf(body,
"<StorageClass>%s</StorageClass></Transition>",
rule->transition->storage_class);
}
if (rule->expiration) {
g_string_append(body, "<Expiration>");
if (rule->expiration->date) {
g_string_append_printf(body, "<Date>%s</Date>",
rule->expiration->date);
} else {
g_string_append_printf(body, "<Days>%u</Days>",
rule->expiration->days);
}
g_string_append(body, "</Expiration>");
}
g_string_append_printf(body, "</Rule>");
}
g_string_append(body, "</LifecycleConfiguration>");
buf.buffer = g_string_free(body, FALSE);
buf.buffer_len = strlen(buf.buffer);
s3_verbose(hdl, 1);
result = perform_request(hdl, "PUT", bucket, NULL, "lifecycle", NULL,
"application/xml", NULL, NULL,
S3_BUFFER_READ_FUNCS, &buf,
NULL, NULL, NULL,
NULL, NULL, result_handling, FALSE);
return result == S3_RESULT_OK;
}
char *
am_strrmspace(
char *str)
{
char *s, *t;
for(s=str,t=str; *s != '\0'; s++) {
if (*s != ' ') {
*t++ = *s;
}
}
*t = '\0';
return t;
}