/*

 * Copyright (C) 2016 Colin Walters <walters@verbum.org>

 *

 * SPDX-License-Identifier: LGPL-2.0+

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, write to the

 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,

 * Boston, MA 02111-1307, USA.

 */

#include "config.h"

#include <gio/gfiledescriptorbased.h>

#include <gio/gunixoutputstream.h>

#include <glib-unix.h>

#include <curl/curl.h>

/* These macros came from 7.43.0, but we want to check

 * for versions a bit earlier than that (to work on CentOS 7),

 * so define them here if we're using an older version.

 */

#ifndef CURL_VERSION_BITS

#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z)

#endif

#ifndef CURL_AT_LEAST_VERSION

#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z))

#endif

/* Cargo culted from https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c */

#ifndef CURLPIPE_MULTIPLEX

/* This little trick will just make sure that we don't enable pipelining for

   libcurls old enough to not have this symbol. It is _not_ defined to zero in

   a recent libcurl header. */

#define CURLPIPE_MULTIPLEX 0

#endif

#include "ostree-fetcher.h"

#include "ostree-fetcher-util.h"

#include "ostree-enumtypes.h"

#include "ostree-repo-private.h"

#include "otutil.h"

#include "ostree-soup-uri.h"

typedef struct FetcherRequest FetcherRequest;

typedef struct SockInfo SockInfo;

static int sock_cb (CURL *e, curl_socket_t s, int what, void *cbp, void *sockp);

static gboolean timer_cb (gpointer data);

static void sock_unref (SockInfo *f);

static int update_timeout_cb (CURLM *multi, long timeout_ms, void *userp);

static void request_unref (FetcherRequest *req);

static void initiate_next_curl_request (FetcherRequest *req, GTask *task);

static void destroy_and_unref_source (GSource *source);

struct OstreeFetcher

{

  GObject parent_instance;

  OstreeFetcherConfigFlags config_flags;

  char *remote_name;

  char *tls_ca_db_path;

  char *tls_client_cert_path;

  char *tls_client_key_path;

  char *cookie_jar_path;

  char *proxy;

  struct curl_slist *extra_headers;

  int tmpdir_dfd;

  char *custom_user_agent;

  GMainContext *mainctx;

  CURLM *multi;

  GSource *timer_event;

  int curl_running;

  GHashTable *outstanding_requests; /* Set<GTask> */

  GHashTable *sockets; /* Set<SockInfo> */

  guint64 bytes_transferred;

};

/* Information associated with a request */

struct FetcherRequest {

  guint refcount;

  GPtrArray *mirrorlist;

  guint idx;

  char *filename;

  guint64 current_size;

  guint64 max_size;

  OstreeFetcherRequestFlags flags;

  gboolean is_membuf;

  GError *caught_write_error;

  GLnxTmpfile tmpf;

  GString *output_buf;

  CURL *easy;

  char error[CURL_ERROR_SIZE];

  OstreeFetcher *fetcher;

};

/* Information associated with a specific socket */

struct SockInfo {

  guint refcount;

  curl_socket_t sockfd;

  int action;

  long timeout;

  GSource *ch;

  OstreeFetcher *fetcher;

};

enum {

  PROP_0,

  PROP_CONFIG_FLAGS

};

G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT)

static void

_ostree_fetcher_set_property (GObject      *object,

                              guint         prop_id,

                              const GValue *value,

                              GParamSpec   *pspec)

{

  OstreeFetcher *self = OSTREE_FETCHER (object);

  switch (prop_id)

    {

      case PROP_CONFIG_FLAGS:

        self->config_flags = g_value_get_flags (value);

        break;

      default:

        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);

        break;

    }

}

static void

_ostree_fetcher_get_property (GObject    *object,

                              guint       prop_id,

                              GValue     *value,

                              GParamSpec *pspec)

{

  OstreeFetcher *self = OSTREE_FETCHER (object);

  switch (prop_id)

    {

      case PROP_CONFIG_FLAGS:

        g_value_set_flags (value, self->config_flags);

        break;

      default:

        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);

        break;

    }

}

static void

_ostree_fetcher_finalize (GObject *object)

{

  OstreeFetcher *self = OSTREE_FETCHER (object);

  curl_multi_cleanup (self->multi);

  g_free (self->remote_name);

  g_free (self->tls_ca_db_path);

  g_free (self->tls_client_cert_path);

  g_free (self->tls_client_key_path);

  g_free (self->cookie_jar_path);

  g_free (self->proxy);

  g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0);

  g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all);

  g_hash_table_unref (self->outstanding_requests);

  g_hash_table_unref (self->sockets);

  g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source);

  if (self->mainctx)

    g_main_context_unref (self->mainctx);

  g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free);

  G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object);

}

static void

_ostree_fetcher_constructed (GObject *object)

{

  // OstreeFetcher *self = OSTREE_FETCHER (object);

  G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object);

}

static void

_ostree_fetcher_class_init (OstreeFetcherClass *klass)

{

  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);

  gobject_class->set_property = _ostree_fetcher_set_property;

  gobject_class->get_property = _ostree_fetcher_get_property;

  gobject_class->finalize = _ostree_fetcher_finalize;

  gobject_class->constructed = _ostree_fetcher_constructed;

  g_object_class_install_property (gobject_class,

                                   PROP_CONFIG_FLAGS,

                                   g_param_spec_flags ("config-flags",

                                                       "",

                                                       "",

                                                       OSTREE_TYPE_FETCHER_CONFIG_FLAGS,

                                                       OSTREE_FETCHER_FLAGS_NONE,

                                                       G_PARAM_READWRITE |

                                                       G_PARAM_CONSTRUCT_ONLY |

                                                       G_PARAM_STATIC_STRINGS));

}

static void

_ostree_fetcher_init (OstreeFetcher *self)

{

  self->multi = curl_multi_init();

  self->outstanding_requests = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)g_object_unref, NULL);

  self->sockets = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)sock_unref, NULL);

  curl_multi_setopt (self->multi, CURLMOPT_SOCKETFUNCTION, sock_cb);

  curl_multi_setopt (self->multi, CURLMOPT_SOCKETDATA, self);

  curl_multi_setopt (self->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb);

  curl_multi_setopt (self->multi, CURLMOPT_TIMERDATA, self);

#if CURL_AT_LEAST_VERSION(7, 30, 0)

  /* Let's do something reasonable here. */

  curl_multi_setopt (self->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 8);

#endif

  /* This version mirrors the version at which we're enabling HTTP2 support.

   * See also https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c

   */

#if CURL_AT_LEAST_VERSION(7, 51, 0)

  curl_multi_setopt (self->multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);

#endif

}

OstreeFetcher *

_ostree_fetcher_new (int                      tmpdir_dfd,

                     const char              *remote_name,

                     OstreeFetcherConfigFlags flags)

{

  OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL);

  fetcher->remote_name = g_strdup (remote_name);

  fetcher->tmpdir_dfd = tmpdir_dfd;

  return fetcher;

}

static void

destroy_and_unref_source (GSource *source)

{

  g_source_destroy (source);

  g_source_unref (source);

}

static char *

request_get_uri (FetcherRequest *req, SoupURI *baseuri)

{

  if (!req->filename)

    return soup_uri_to_string (baseuri, FALSE);

  { g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE);

    return g_build_filename (uristr, req->filename, NULL);

  }

}

static gboolean

ensure_tmpfile (FetcherRequest *req, GError **error)

{

  if (!req->tmpf.initialized)

    {

      if (!_ostree_fetcher_tmpf_from_flags (req->flags, req->fetcher->tmpdir_dfd,

                                            &req->tmpf, error))

        return FALSE;

    }

  return TRUE;

}

/* Check for completed transfers, and remove their easy handles */

static void

check_multi_info (OstreeFetcher *fetcher)

{

  CURLMsg *msg;

  int msgs_left;

  while ((msg = curl_multi_info_read (fetcher->multi, &msgs_left)) != NULL)

    {

      long response;

      CURL *easy = msg->easy_handle;

      CURLcode curlres = msg->data.result;

      GTask *task;

      FetcherRequest *req;

      const char *eff_url;

      gboolean is_file;

      gboolean continued_request = FALSE;

      if (msg->msg != CURLMSG_DONE)

        continue;

      curl_easy_getinfo (easy, CURLINFO_PRIVATE, &task);

      curl_easy_getinfo (easy, CURLINFO_EFFECTIVE_URL, &eff_url);

      /* We should have limited the protocols; this is what

       * curl's tool_operate.c does.

       */

      is_file = g_str_has_prefix (eff_url, "file:");

      g_assert (is_file || g_str_has_prefix (eff_url, "http"));

      req = g_task_get_task_data (task);

      if (req->caught_write_error)

        g_task_return_error (task, g_steal_pointer (&req->caught_write_error));

      else if (curlres != CURLE_OK)

        {

          if (is_file && curlres == CURLE_FILE_COULDNT_READ_FILE)

            {

              /* Handle file not found */

              g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,

                                       "%s", curl_easy_strerror (curlres));

            }

          else

            {

              g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED,

                                       "While fetching %s: [%u] %s", eff_url, curlres,

                                       curl_easy_strerror (curlres));

              _ostree_fetcher_journal_failure (req->fetcher->remote_name,

                                               eff_url, curl_easy_strerror (curlres));

            }

        }

      else

        {

          curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);

          if (!is_file && !(response >= 200 && response < 300))

            {

              GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response);

              if (req->idx + 1 == req->mirrorlist->len)

                {

                  g_autofree char *msg = g_strdup_printf ("Server returned HTTP %lu", response);

                  g_task_return_new_error (task, G_IO_ERROR, giocode,

                                           "%s", msg);

                  if (req->fetcher->remote_name &&

                      !((req->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 &&

                        giocode == G_IO_ERROR_NOT_FOUND))

                    _ostree_fetcher_journal_failure (req->fetcher->remote_name,

                                                     eff_url, msg);

                }

              else

                {

                  continued_request = TRUE;

                }

            }

          else if (req->is_membuf)

            {

              GBytes *ret;

              if ((req->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0)

                g_string_append_c (req->output_buf, '\0');

              ret = g_string_free_to_bytes (req->output_buf);

              req->output_buf = NULL;

              g_task_return_pointer (task, ret, (GDestroyNotify)g_bytes_unref);

            }

          else

            {

              g_autoptr(GError) local_error = NULL;

              GError **error = &local_error;

              if (!ensure_tmpfile (req, error))

                {

                  g_task_return_error (task, g_steal_pointer (&local_error));

                }

              else if (lseek (req->tmpf.fd, 0, SEEK_SET) < 0)

                {

                  glnx_set_error_from_errno (error);

                  g_task_return_error (task, g_steal_pointer (&local_error));

                }

              else

                {

                  /* We return the tmpfile in the _finish wrapper */

                  g_task_return_boolean (task, TRUE);

                }

            }

        }

      curl_multi_remove_handle (fetcher->multi, easy);

      if (continued_request)

        {

          req->idx++;

          initiate_next_curl_request (req, task);

        }

      else

        {

          g_hash_table_remove (fetcher->outstanding_requests, task);

          if (g_hash_table_size (fetcher->outstanding_requests) == 0)

            {

              g_clear_pointer (&fetcher->mainctx, g_main_context_unref);

            }

        }

    }

}

/* Called by glib when our timeout expires */

static gboolean

timer_cb (gpointer data)

{

  OstreeFetcher *fetcher = data;

  GSource *orig_src = fetcher->timer_event;

  (void)curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running);

  check_multi_info (fetcher);

  if (fetcher->timer_event == orig_src)

    fetcher->timer_event = NULL;

  return FALSE;

}

/* Update the event timer after curl_multi library calls */

static int

update_timeout_cb (CURLM *multi, long timeout_ms, void *userp)

{

  OstreeFetcher *fetcher = userp;

  g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source);

  if (timeout_ms != -1)

    {

      fetcher->timer_event = g_timeout_source_new (timeout_ms);

      g_source_set_callback (fetcher->timer_event, timer_cb, fetcher, NULL);

      g_source_attach (fetcher->timer_event, fetcher->mainctx);

    }

  return 0;

}

/* Called by glib when we get action on a multi socket */

static gboolean

event_cb (int fd, GIOCondition condition, gpointer data)

{

  OstreeFetcher *fetcher = data;

  int action =

    (condition & G_IO_IN ? CURL_CSELECT_IN : 0) |

    (condition & G_IO_OUT ? CURL_CSELECT_OUT : 0);

  (void)curl_multi_socket_action (fetcher->multi, fd, action, &fetcher->curl_running);

  check_multi_info (fetcher);

  if (fetcher->curl_running > 0)

    {

      return TRUE;

    }

  else

    {

      return FALSE;

    }

}

/* Clean up the SockInfo structure */

static void

sock_unref (SockInfo *f)

{

  if (!f)

    return;

  if (--f->refcount != 0)

    return;

  g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source);

  g_free (f);

}

/* Assign information to a SockInfo structure */

static void

setsock (SockInfo*f, curl_socket_t s, int act, OstreeFetcher *fetcher)

{

  GIOCondition kind =

     (act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0);

  f->sockfd = s;

  f->action = act;

  g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source);

  /* TODO - investigate new g_source_modify_unix_fd() so changing the poll

   * flags involves less allocation.

   */

  f->ch = g_unix_fd_source_new (f->sockfd, kind);

  g_source_set_callback (f->ch, (GSourceFunc) event_cb, fetcher, NULL);

  g_source_attach (f->ch, fetcher->mainctx);

}

/* Initialize a new SockInfo structure */

static void

addsock (curl_socket_t s, CURL *easy, int action, OstreeFetcher *fetcher)

{

  SockInfo *fdp = g_new0 (SockInfo, 1);

  fdp->refcount = 1;

  fdp->fetcher = fetcher;

  setsock (fdp, s, action, fetcher);

  curl_multi_assign (fetcher->multi, s, fdp);

  g_hash_table_add (fetcher->sockets, fdp);

}

/* CURLMOPT_SOCKETFUNCTION */

static int

sock_cb (CURL *easy, curl_socket_t s, int what, void *cbp, void *sockp)

{

  OstreeFetcher *fetcher = cbp;

  SockInfo *fdp = (SockInfo*) sockp;

  if (what == CURL_POLL_REMOVE)

    {

      if (!g_hash_table_remove (fetcher->sockets, fdp))

        g_assert_not_reached ();

    }

  else

    {

      if (!fdp)

        {

          addsock (s, easy, what, fetcher);

        }

      else

        {

          setsock (fdp, s, what, fetcher);

        }

    }

  return 0;

}

/* CURLOPT_WRITEFUNCTION */

static size_t

write_cb (void *ptr, size_t size, size_t nmemb, void *data)

{

  const size_t realsize = size * nmemb;

  GTask *task = data;

  FetcherRequest *req;

  req = g_task_get_task_data (task);

  if (req->caught_write_error)

    return -1;

  if (req->max_size > 0)

    {

      if (realsize > req->max_size ||

          (realsize + req->current_size) > req->max_size)

        {

          const char *eff_url;

          curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url);

          req->caught_write_error =  g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,

                                                  "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",

                                                  eff_url, req->max_size);

          return -1;

        }

    }

  if (req->is_membuf)

    g_string_append_len (req->output_buf, ptr, realsize);

  else

    {

      if (!ensure_tmpfile (req, &req->caught_write_error))

        return -1;

      g_assert (req->tmpf.fd >= 0);

      if (glnx_loop_write (req->tmpf.fd, ptr, realsize) < 0)

        {

          glnx_set_error_from_errno (&req->caught_write_error);

          return -1;

        }

    }

  req->current_size += realsize;

  req->fetcher->bytes_transferred += realsize;

  return realsize;

}

/* CURLOPT_PROGRESSFUNCTION */

static int

prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)

{

  GTask *task = p;

  FetcherRequest *req;

  char *eff_url;

  req = g_task_get_task_data (task);

  curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url);

  g_printerr ("Progress: %s (%g/%g)\n", eff_url, dlnow, dltotal);

  return 0;

}

static void

request_unref (FetcherRequest *req)

{

  if (--req->refcount != 0)

    return;

  g_ptr_array_unref (req->mirrorlist);

  g_free (req->filename);

  g_clear_error (&req->caught_write_error);

  glnx_tmpfile_clear (&req->tmpf);

  if (req->output_buf)

    g_string_free (req->output_buf, TRUE);

  curl_easy_cleanup (req->easy);

  g_free (req);

}

int

_ostree_fetcher_get_dfd (OstreeFetcher *fetcher)

{

  return fetcher->tmpdir_dfd;

}

void

_ostree_fetcher_set_proxy (OstreeFetcher *self,

                           const char    *http_proxy)

{

  g_free (self->proxy);

  self->proxy = g_strdup (http_proxy);

}

void

_ostree_fetcher_set_cookie_jar (OstreeFetcher *self,

                                const char    *jar_path)

{

  g_free (self->cookie_jar_path);

  self->cookie_jar_path = g_strdup (jar_path);

}

void

_ostree_fetcher_set_client_cert (OstreeFetcher   *self,

                                 const char      *cert_path,

                                 const char      *key_path)

{

  g_assert ((cert_path == NULL && key_path == NULL)

            || (cert_path != NULL && key_path != NULL));

  g_free (self->tls_client_cert_path);

  self->tls_client_cert_path = g_strdup (cert_path);

  g_free (self->tls_client_key_path);

  self->tls_client_key_path = g_strdup (key_path);

}

void

_ostree_fetcher_set_tls_database (OstreeFetcher *self,

                                  const char    *dbpath)

{

  g_free (self->tls_ca_db_path);

  self->tls_ca_db_path = g_strdup (dbpath);

}

void

_ostree_fetcher_set_extra_headers (OstreeFetcher *self,

                                   GVariant      *extra_headers)

{

  GVariantIter viter;

  const char *key;

  const char *value;

  g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all);

  g_variant_iter_init (&viter, extra_headers);

  while (g_variant_iter_loop (&viter, "(&s&s)", &key, &value))

    {

      g_autofree char *header = g_strdup_printf ("%s: %s", key, value);

      self->extra_headers = curl_slist_append (self->extra_headers, header);

    }

}

void

_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self,

                                      const char    *extra_user_agent)

{

  g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free);

  if (extra_user_agent)

    {

      self->custom_user_agent =

        g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent);

    }

}

/* Re-bind all of the outstanding curl items to our new main context */

static void

adopt_steal_mainctx (OstreeFetcher *self,

                     GMainContext *mainctx)

{

  g_assert (self->mainctx == NULL);

  self->mainctx = mainctx; /* Transfer */

  if (self->timer_event != NULL)

    {

      guint64 readytime = g_source_get_ready_time (self->timer_event);

      guint64 curtime = g_source_get_time (self->timer_event);

      guint64 timeout_micros = curtime - readytime;

      if (curtime < readytime)

        timeout_micros = 0;

      update_timeout_cb (self->multi, timeout_micros / 1000, self);

    }

  GLNX_HASH_TABLE_FOREACH (self->sockets, SockInfo*, fdp)

    setsock (fdp, fdp->sockfd, fdp->action, self);

}

static void

initiate_next_curl_request (FetcherRequest *req,

                            GTask *task)

{

  CURLcode rc;

  OstreeFetcher *self = req->fetcher;

  if (req->easy)

    curl_easy_cleanup (req->easy);

  req->easy = curl_easy_init ();

  g_assert (req->easy);

  g_assert_cmpint (req->idx, <, req->mirrorlist->len);

  SoupURI *baseuri = req->mirrorlist->pdata[req->idx];

  { g_autofree char *uri = request_get_uri (req, baseuri);

    curl_easy_setopt (req->easy, CURLOPT_URL, uri);

  }

  curl_easy_setopt (req->easy, CURLOPT_USERAGENT,

                    self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING);

  if (self->extra_headers)

    curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);

  if (self->cookie_jar_path)

    {

      rc = curl_easy_setopt (req->easy, CURLOPT_COOKIEFILE, self->cookie_jar_path);

      g_assert_cmpint (rc, ==, CURLM_OK);

      rc = curl_easy_setopt (req->easy, CURLOPT_COOKIELIST, "RELOAD");

      g_assert_cmpint (rc, ==, CURLM_OK);

    }

  if (self->proxy)

    {

      rc = curl_easy_setopt (req->easy, CURLOPT_PROXY, self->proxy);

      g_assert_cmpint (rc, ==, CURLM_OK);

    }

  if (self->tls_ca_db_path)

    curl_easy_setopt (req->easy, CURLOPT_CAINFO, self->tls_ca_db_path);

  if ((self->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0)

    curl_easy_setopt (req->easy, CURLOPT_SSL_VERIFYPEER, 0L);

  if (self->tls_client_cert_path)

    {

      /* Support for pkcs11:

       * https://github.com/ostreedev/ostree/pull/1183

       * This will be used by https://github.com/advancedtelematic/aktualizr

       * at least to fetch certificates.  No test coverage at the moment

       * though. See https://gitlab.com/gnutls/gnutls/tree/master/tests/pkcs11

       * and https://github.com/opendnssec/SoftHSMv2 and

       * https://github.com/p11-glue/p11-kit/tree/master/p11-kit for

       * possible ideas there.

       */

      if (g_str_has_prefix (self->tls_client_key_path, "pkcs11:"))

        {

          curl_easy_setopt (req->easy, CURLOPT_SSLENGINE, "pkcs11");

          curl_easy_setopt (req->easy, CURLOPT_SSLENGINE_DEFAULT, 1L);

          curl_easy_setopt (req->easy, CURLOPT_SSLKEYTYPE, "ENG");

        }

      if (g_str_has_prefix (self->tls_client_cert_path, "pkcs11:"))

        curl_easy_setopt (req->easy, CURLOPT_SSLCERTTYPE, "ENG");

      curl_easy_setopt (req->easy, CURLOPT_SSLCERT, self->tls_client_cert_path);

      curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path);

    }

  if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) > 0)

    curl_easy_setopt (req->easy, CURLOPT_ACCEPT_ENCODING, "");

  /* If we have e.g. basic auth in the URL string, let's honor that */

  const char *username = soup_uri_get_user (baseuri);

  curl_easy_setopt (req->easy, CURLOPT_USERNAME, username);

  const char *password = soup_uri_get_password (baseuri);

  curl_easy_setopt (req->easy, CURLOPT_PASSWORD, password);

  /* We should only speak HTTP; TODO: only enable file if specified */

  curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE));

  /* Picked the current version in F25 as of 20170127, since

   * there are numerous HTTP/2 fixes since the original version in

   * libcurl 7.43.0.

   */

  if (!(self->config_flags & OSTREE_FETCHER_FLAGS_DISABLE_HTTP2))

    {

#if CURL_AT_LEAST_VERSION(7, 51, 0)

      curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);

#endif

      /* https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c */

#if (CURLPIPE_MULTIPLEX > 0)

      /* wait for pipe connection to confirm */

      curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L);

#endif

    }

  curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);

  if (g_getenv ("OSTREE_DEBUG_HTTP"))

    curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);

  curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error);

  /* Note that the "easy" object's privdata is the task */

  curl_easy_setopt (req->easy, CURLOPT_NOPROGRESS, 1L);

  curl_easy_setopt (req->easy, CURLOPT_PROGRESSFUNCTION, prog_cb);

  curl_easy_setopt (req->easy, CURLOPT_FOLLOWLOCATION, 1L);

  curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L);

  /* We used to set CURLOPT_LOW_SPEED_LIMIT and CURLOPT_LOW_SPEED_TIME

   * here, but see https://github.com/ostreedev/ostree/issues/878#issuecomment-347228854

   * basically those options don't play well with HTTP2 at the moment

   * where we can have lots of outstanding requests.  Further,

   * we could implement that functionality at a higher level

   * more consistently too.

   */

  /* closure bindings -> task */

  curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);

  curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);

  curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);

  CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy);

  g_assert (multi_rc == CURLM_OK);

}

static void

_ostree_fetcher_request_async (OstreeFetcher         *self,

                               GPtrArray             *mirrorlist,

                               const char            *filename,

                               OstreeFetcherRequestFlags flags,

                               gboolean               is_membuf,

                               guint64                max_size,

                               int                    priority,

                               GCancellable          *cancellable,

                               GAsyncReadyCallback    callback,

                               gpointer               user_data)

{

  g_autoptr(GTask) task = NULL;

  FetcherRequest *req;

  g_autoptr(GMainContext) mainctx = g_main_context_ref_thread_default ();

  /* We don't support multiple concurrent main contexts; take

   * a ref to the first one, and require that later invocations

   * share it.

   */

  if (g_hash_table_size (self->outstanding_requests) == 0

      && mainctx != self->mainctx)

    {

      adopt_steal_mainctx (self, g_steal_pointer (&mainctx));

    }

  else

    {

      g_assert (self->mainctx == mainctx);

    }

  req = g_new0 (FetcherRequest, 1);

  req->refcount = 1;

  req->error[0]='\0';

  req->fetcher = self;

  req->mirrorlist = g_ptr_array_ref (mirrorlist);

  req->filename = g_strdup (filename);

  req->max_size = max_size;

  req->flags = flags;

  req->is_membuf = is_membuf;

  /* We'll allocate the tmpfile on demand, so we handle

   * file I/O errors just in the write func.

   */

  if (req->is_membuf)

    req->output_buf = g_string_new ("");

  task = g_task_new (self, cancellable, callback, user_data);

  /* We'll use the GTask priority for our own priority queue. */

  g_task_set_priority (task, priority);

  g_task_set_source_tag (task, _ostree_fetcher_request_async);

  g_task_set_task_data (task, req, (GDestroyNotify) request_unref);

  initiate_next_curl_request (req, task);

  g_hash_table_add (self->outstanding_requests, g_steal_pointer (&task));

}

void