/*
* Copyright (C) 2011,2012,2013 Colin Walters <walters@verbum.org>
* Copyright © 2017 Endless Mobile, Inc.
*
* SPDX-License-Identifier: LGPL-2.0+
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Authors:
* - Colin Walters <walters@verbum.org>
* - Philip Withnall <withnall@endlessm.com>
*/
#include "config.h"
#include "libglnx.h"
#include "ostree.h"
#include "otutil.h"
#include "ostree-repo-pull-private.h"
#ifdef HAVE_LIBCURL_OR_LIBSOUP
#include "ostree-core-private.h"
#include "ostree-repo-static-delta-private.h"
#include "ostree-metalink.h"
#include "ostree-repo-finder.h"
#include "ostree-repo-finder-config.h"
#include "ostree-repo-finder-mount.h"
#ifdef HAVE_AVAHI
#include "ostree-repo-finder-avahi.h"
#endif /* HAVE_AVAHI */
#include <gio/gunixinputstream.h>
#include <sys/statvfs.h>
#ifdef HAVE_LIBSYSTEMD
#include <systemd/sd-journal.h>
#endif
#define OSTREE_MESSAGE_FETCH_COMPLETE_ID SD_ID128_MAKE(75,ba,3d,eb,0a,f0,41,a9,a4,62,72,ff,85,d9,e7,3e)
#define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY)
#define OSTREE_REPO_PULL_METADATA_PRIORITY (OSTREE_REPO_PULL_CONTENT_PRIORITY - 100)
/* Arbitrarily chosen number of retries for all download operations when they
* receive a transient network error (such as a socket timeout) — see
* _ostree_fetcher_should_retry_request(). This is the default value for the
* `n-network-retries` pull option. */
#define DEFAULT_N_NETWORK_RETRIES 5
typedef struct {
OtPullData *pull_data;
GVariant *object;
char *path;
gboolean is_detached_meta;
/* Only relevant when is_detached_meta is TRUE. Controls
* whether to fetch the primary object after fetching its
* detached metadata (no need if it's already stored). */
gboolean object_is_stored;
OstreeCollectionRef *requested_ref; /* (nullable) */
guint n_retries_remaining;
} FetchObjectData;
typedef struct {
OtPullData *pull_data;
GVariant *objects;
char *expected_checksum;
char *from_revision;
char *to_revision;
guint i;
guint64 size;
guint n_retries_remaining;
} FetchStaticDeltaData;
typedef struct {
guchar csum[OSTREE_SHA256_DIGEST_LEN];
char *path;
OstreeObjectType objtype;
guint recursion_depth; /* NB: not used anymore, though might be nice to print */
OstreeCollectionRef *requested_ref; /* (nullable) */
} ScanObjectQueueData;
typedef struct {
OtPullData *pull_data;
char *from_revision;
char *to_revision;
OstreeCollectionRef *requested_ref; /* (nullable) */
guint n_retries_remaining;
} FetchDeltaSuperData;
static void
variant_or_null_unref (gpointer data)
{
if (data)
g_variant_unref (data);
}
static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch);
static void start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch);
static void start_fetch_delta_superblock (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data);
static gboolean fetcher_queue_is_full (OtPullData *pull_data);
static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref);
static void queue_scan_one_metadata_object_s (OtPullData *pull_data,
ScanObjectQueueData *scan_data);
static void queue_scan_one_metadata_object_c (OtPullData *pull_data,
const guchar *csum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref);
static void enqueue_one_object_request_s (OtPullData *pull_data,
FetchObjectData *fetch_data);
static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data);
static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data,
FetchStaticDeltaData *fetch_data);
static gboolean scan_one_metadata_object (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref,
GCancellable *cancellable,
GError **error);
static void scan_object_queue_data_free (ScanObjectQueueData *scan_data);
static gboolean
update_progress (gpointer user_data)
{
OtPullData *pull_data;
guint outstanding_writes;
guint outstanding_fetches;
guint64 bytes_transferred;
guint fetched;
guint requested;
guint n_scanned_metadata;
guint64 start_time;
pull_data = user_data;
if (! pull_data->progress)
return FALSE;
/* In dry run, we only emit progress once metadata is done */
if (pull_data->dry_run && pull_data->n_outstanding_metadata_fetches > 0)
return TRUE;
outstanding_writes = pull_data->n_outstanding_content_write_requests +
pull_data->n_outstanding_metadata_write_requests +
pull_data->n_outstanding_deltapart_write_requests;
outstanding_fetches = pull_data->n_outstanding_content_fetches +
pull_data->n_outstanding_metadata_fetches +
pull_data->n_outstanding_deltapart_fetches;
bytes_transferred = _ostree_fetcher_bytes_transferred (pull_data->fetcher);
fetched = pull_data->n_fetched_metadata + pull_data->n_fetched_content;
requested = pull_data->n_requested_metadata + pull_data->n_requested_content;
n_scanned_metadata = pull_data->n_scanned_metadata;
start_time = pull_data->start_time;
ostree_async_progress_set (pull_data->progress,
"outstanding-fetches", "u", outstanding_fetches,
"outstanding-writes", "u", outstanding_writes,
"fetched", "u", fetched,
"requested", "u", requested,
"scanning", "u", g_queue_is_empty (&pull_data->scan_object_queue) ? 0 : 1,
"caught-error", "b", pull_data->caught_error,
"scanned-metadata", "u", n_scanned_metadata,
"bytes-transferred", "t", bytes_transferred,
"start-time", "t", start_time,
/* We use these status keys even though we now also
* use these values for filesystem-local pulls.
*/
"metadata-fetched-localcache", "u", pull_data->n_imported_metadata,
"content-fetched-localcache", "u", pull_data->n_imported_content,
/* Deltas */
"fetched-delta-parts",
"u", pull_data->n_fetched_deltaparts,
"total-delta-parts",
"u", pull_data->n_total_deltaparts,
"fetched-delta-fallbacks",
"u", pull_data->n_fetched_deltapart_fallbacks,
"total-delta-fallbacks",
"u", pull_data->n_total_delta_fallbacks,
"fetched-delta-part-size",
"t", pull_data->fetched_deltapart_size,
"total-delta-part-size",
"t", pull_data->total_deltapart_size,
"total-delta-part-usize",
"t", pull_data->total_deltapart_usize,
"total-delta-superblocks",
"u", pull_data->static_delta_superblocks->len,
/* We fetch metadata before content. These allow us to report metadata fetch progress specifically. */
"outstanding-metadata-fetches", "u", pull_data->n_outstanding_metadata_fetches,
"metadata-fetched", "u", pull_data->n_fetched_metadata,
/* Overall status. */
"status", "s", "",
NULL);
if (pull_data->dry_run)
pull_data->dry_run_emitted_progress = TRUE;
return TRUE;
}
/* The core logic function for whether we should continue the main loop */
static gboolean
pull_termination_condition (OtPullData *pull_data)
{
gboolean current_fetch_idle = (pull_data->n_outstanding_metadata_fetches == 0 &&
pull_data->n_outstanding_content_fetches == 0 &&
pull_data->n_outstanding_deltapart_fetches == 0);
gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 &&
pull_data->n_outstanding_content_write_requests == 0 &&
pull_data->n_outstanding_deltapart_write_requests == 0 );
gboolean current_scan_idle = g_queue_is_empty (&pull_data->scan_object_queue);
gboolean current_idle = current_fetch_idle && current_write_idle && current_scan_idle;
/* we only enter the main loop when we're fetching objects */
g_assert (pull_data->phase == OSTREE_PULL_PHASE_FETCHING_OBJECTS);
if (pull_data->dry_run)
return pull_data->dry_run_emitted_progress;
if (current_idle)
g_debug ("pull: idle, exiting mainloop");
return current_idle;
}
/* Most async operations finish by calling this function; it will consume
* @errorp if set, update statistics, and initiate processing of any further
* requests as appropriate.
*/
static void
check_outstanding_requests_handle_error (OtPullData *pull_data,
GError **errorp)
{
g_assert (errorp);
GError *error = *errorp;
if (error)
{
g_debug ("Request caught error: %s", error->message);
if (!pull_data->caught_error)
{
pull_data->caught_error = TRUE;
g_propagate_error (pull_data->async_error, g_steal_pointer (errorp));
}
else
{
g_clear_error (errorp);
}
}
/* If we're in error state, we wait for any pending operations to complete,
* but ensure that all no further operations are queued.
*/
if (pull_data->caught_error)
{
g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);
g_queue_clear (&pull_data->scan_object_queue);
g_hash_table_remove_all (pull_data->pending_fetch_metadata);
g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks);
g_hash_table_remove_all (pull_data->pending_fetch_deltaparts);
g_hash_table_remove_all (pull_data->pending_fetch_content);
}
else
{
GHashTableIter hiter;
gpointer key, value;
/* We may have just completed an async fetch operation. Now we look at
* possibly enqueuing more requests. The goal of queuing is to both avoid
* overloading the fetcher backend with HTTP requests, but also to
* prioritize metadata fetches over content, so we have accurate
* reporting. Hence here, we process metadata fetches first.
*/
/* Try filling the queue with metadata we need to fetch */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_metadata);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
GVariant *objname = key;
FetchObjectData *fetch = value;
/* Steal both key and value */
g_hash_table_iter_steal (&hiter);
/* This takes ownership of the value */
start_fetch (pull_data, fetch);
/* And unref the key */
g_variant_unref (objname);
}
/* Next, process delta superblock requests */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
FetchDeltaSuperData *fetch = key;
g_hash_table_iter_steal (&hiter);
start_fetch_delta_superblock (pull_data, g_steal_pointer (&fetch));
}
/* Now, process deltapart requests */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_deltaparts);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
FetchStaticDeltaData *fetch = key;
g_hash_table_iter_steal (&hiter);
/* Takes ownership */
start_fetch_deltapart (pull_data, fetch);
}
/* Next, fill the queue with content */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_content);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
char *checksum = key;
FetchObjectData *fetch = value;
/* Steal both key and value */
g_hash_table_iter_steal (&hiter);
/* This takes ownership of the value */
start_fetch (pull_data, fetch);
/* And unref the key */
g_free (checksum);
}
}
}
/* We have a total-request limit, as well has a hardcoded max of 2 for delta
* parts. The logic for the delta one is that processing them is expensive, and
* doing multiple simultaneously could risk space/memory on smaller devices. We
* also throttle on outstanding writes in case fetches are faster.
*/
static gboolean
fetcher_queue_is_full (OtPullData *pull_data)
{
const gboolean fetch_full =
((pull_data->n_outstanding_metadata_fetches +
pull_data->n_outstanding_content_fetches +
pull_data->n_outstanding_deltapart_fetches) ==
_OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS);
const gboolean deltas_full =
(pull_data->n_outstanding_deltapart_fetches ==
_OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
const gboolean writes_full =
((pull_data->n_outstanding_metadata_write_requests +
pull_data->n_outstanding_content_write_requests +
pull_data->n_outstanding_deltapart_write_requests) >=
_OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
return fetch_full || deltas_full || writes_full;
}
static void
scan_object_queue_data_free (ScanObjectQueueData *scan_data)
{
g_free (scan_data->path);
if (scan_data->requested_ref != NULL)
ostree_collection_ref_free (scan_data->requested_ref);
g_free (scan_data);
}
/* Called out of the main loop to process the "scan object queue", which is a
* queue of metadata objects (commits and dirtree, but not dirmeta) to parse to
* look for further objects. Basically wraps execution of
* `scan_one_metadata_object()`.
*/
static gboolean
idle_worker (gpointer user_data)
{
OtPullData *pull_data = user_data;
ScanObjectQueueData *scan_data;
g_autoptr(GError) error = NULL;
scan_data = g_queue_pop_head (&pull_data->scan_object_queue);
if (!scan_data)
{
g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
return G_SOURCE_REMOVE;
}
char checksum[OSTREE_SHA256_STRING_LEN+1];
ostree_checksum_inplace_from_bytes (scan_data->csum, checksum);
scan_one_metadata_object (pull_data, checksum, scan_data->objtype,
scan_data->path, scan_data->recursion_depth,
scan_data->requested_ref, pull_data->cancellable, &error);
/* No need to retry scan tasks, since they’re local. */
check_outstanding_requests_handle_error (pull_data, &error);
scan_object_queue_data_free (scan_data);
return G_SOURCE_CONTINUE;
}
static void
ensure_idle_queued (OtPullData *pull_data)
{
GSource *idle_src;
if (pull_data->idle_src)
return;
idle_src = g_idle_source_new ();
g_source_set_callback (idle_src, idle_worker, pull_data, NULL);
g_source_attach (idle_src, pull_data->main_context);
pull_data->idle_src = idle_src;
/* Ownership is transferred to pull_data */
g_source_unref (idle_src);
}
typedef struct {
OtPullData *pull_data;
GInputStream *result_stream;
} OstreeFetchUriSyncData;
static gboolean
fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
guint n_network_retries,
char **out_contents,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GBytes) bytes = NULL;
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
n_network_retries,
&bytes,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
return FALSE;
gsize len;
g_autofree char *ret_contents = g_bytes_unref_to_data (g_steal_pointer (&bytes), &len);
if (!g_utf8_validate (ret_contents, -1, NULL))
return glnx_throw (error, "Invalid UTF-8");
ot_transfer_out_value (out_contents, &ret_contents);
return TRUE;
}
static gboolean
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
guint n_network_retries,
char **out_contents,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
NULL, n_network_retries,
out_contents,
cancellable, error);
}
static void
enqueue_one_object_request (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
const char *path,
gboolean is_detached_meta,
gboolean object_is_stored,
const OstreeCollectionRef *ref);
static gboolean
matches_pull_dir (const char *current_file,
const char *pull_dir,
gboolean current_file_is_dir)
{
const char *rest;
if (g_str_has_prefix (pull_dir, current_file))
{
rest = pull_dir + strlen (current_file);
if (*rest == 0)
{
/* The current file is exactly the same as the specified
pull dir. This matches always, even if the file is not a
directory. */
return TRUE;
}
if (*rest == '/')
{
/* The current file is a directory-prefix of the pull_dir.
Match only if this is supposed to be a directory */
return current_file_is_dir;
}
/* Matched a non-directory prefix such as /foo being a prefix of /fooo,
no match */
return FALSE;
}
if (g_str_has_prefix (current_file, pull_dir))
{
rest = current_file + strlen (pull_dir);
/* Only match if the prefix match matched the entire directory
component */
return *rest == '/';
}
return FALSE;
}
static gboolean
pull_matches_subdir (OtPullData *pull_data,
const char *path,
const char *basename,
gboolean basename_is_dir)
{
if (pull_data->dirs == NULL)
return TRUE;
g_autofree char *file = g_strconcat (path, basename, NULL);
for (guint i = 0; i < pull_data->dirs->len; i++)
{
const char *pull_dir = g_ptr_array_index (pull_data->dirs, i);
if (matches_pull_dir (file, pull_dir, basename_is_dir))
return TRUE;
}
return FALSE;
}
typedef struct {
OtPullData *pull_data;
OstreeRepo *src_repo;
char checksum[OSTREE_SHA256_STRING_LEN+1];
} ImportLocalAsyncData;
/* Asynchronously import a single content object. @src_repo is either
* pull_data->remote_repo_local or one of pull_data->localcache_repos.
*/
static void
async_import_in_thread (GTask *task,
gpointer source,
gpointer task_data,
GCancellable *cancellable)
{
ImportLocalAsyncData *iataskdata = task_data;
OtPullData *pull_data = iataskdata->pull_data;
g_autoptr(GError) local_error = NULL;
/* pull_data->importflags was set up in the pull option processing */
if (!_ostree_repo_import_object (pull_data->repo, iataskdata->src_repo,
OSTREE_OBJECT_TYPE_FILE, iataskdata->checksum,
pull_data->importflags, cancellable, &local_error))
g_task_return_error (task, g_steal_pointer (&local_error));
else
g_task_return_boolean (task, TRUE);
}
/* Start an async import of a single object; currently used for content objects.
* @src_repo is from pull_data->remote_repo_local or
* pull_data->localcache_repos.
*
* One important special case here is handling the
* OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES flag.
*/
static void
async_import_one_local_content_object (OtPullData *pull_data,
OstreeRepo *src_repo,
const char *checksum,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
ImportLocalAsyncData *iataskdata = g_new0 (ImportLocalAsyncData, 1);
iataskdata->pull_data = pull_data;
iataskdata->src_repo = src_repo;
memcpy (iataskdata->checksum, checksum, OSTREE_SHA256_STRING_LEN);
g_autoptr(GTask) task = g_task_new (pull_data->repo, cancellable, callback, user_data);
g_task_set_source_tag (task, async_import_one_local_content_object);
g_task_set_task_data (task, iataskdata, g_free);
pull_data->n_outstanding_content_write_requests++;
g_task_run_in_thread (task, async_import_in_thread);
}
static gboolean
async_import_one_local_content_object_finish (OtPullData *pull_data,
GAsyncResult *result,
GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, pull_data->repo), FALSE);
return g_task_propagate_boolean ((GTask*)result, error);
}
static void
on_local_object_imported (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
OtPullData *pull_data = user_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
if (!async_import_one_local_content_object_finish (pull_data, result, error))
goto out;
out:
pull_data->n_imported_content++;
g_assert_cmpint (pull_data->n_outstanding_content_write_requests, >, 0);
pull_data->n_outstanding_content_write_requests--;
/* No retries for local reads. */
check_outstanding_requests_handle_error (pull_data, &local_error);
}
static gboolean
scan_dirtree_object (OtPullData *pull_data,
const char *checksum,
const char *path,
int recursion_depth,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GVariant) tree = NULL;
if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_DIR_TREE, checksum,
&tree, error))
return FALSE;
/* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
g_autoptr(GVariant) files_variant = g_variant_get_child_value (tree, 0);
const guint n = g_variant_n_children (files_variant);
for (guint i = 0; i < n; i++)
{
const char *filename;
gboolean file_is_stored;
g_autoptr(GVariant) csum = NULL;
g_autofree char *file_checksum = NULL;
g_variant_get_child (files_variant, i, "(&s@ay)", &filename, &csum);
/* Note this is now obsoleted by the _ostree_validate_structureof_metadata()
* but I'm keeping this since:
* 1) It's cheap
* 2) We want to continue to do validation for objects written to disk
* before libostree's validation was strengthened.
*/
if (!ot_util_filename_validate (filename, error))
return FALSE;
/* Skip files if we're traversing a request only directory, unless it exactly
* matches the path */
if (!pull_matches_subdir (pull_data, path, filename, FALSE))
continue;
file_checksum = ostree_checksum_from_bytes_v (csum);
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
&file_is_stored, cancellable, error))
return FALSE;
/* If we already have this object, move on to the next */
if (file_is_stored)
continue;
/* Already have a request pending? If so, move on to the next */
if (g_hash_table_lookup (pull_data->requested_content, file_checksum))
continue;
/* Is this a local repo? */
if (pull_data->remote_repo_local)
{
async_import_one_local_content_object (pull_data, pull_data->remote_repo_local,
file_checksum, cancellable,
on_local_object_imported,
pull_data);
g_hash_table_add (pull_data->requested_content, g_steal_pointer (&file_checksum));
/* Note early loop continue */
continue;
}
/* We're doing HTTP, but see if we have the object in a local cache first */
gboolean did_import_from_cache_repo = FALSE;
if (pull_data->localcache_repos)
{
for (guint j = 0; j < pull_data->localcache_repos->len; j++)
{
OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j];
gboolean localcache_repo_has_obj;
if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
&localcache_repo_has_obj, cancellable, error))
return FALSE;
if (!localcache_repo_has_obj)
continue;
async_import_one_local_content_object (pull_data, localcache_repo, file_checksum, cancellable,
on_local_object_imported, pull_data);
g_hash_table_add (pull_data->requested_content, g_steal_pointer (&file_checksum));
did_import_from_cache_repo = TRUE;
break;
}
}
if (did_import_from_cache_repo)
continue; /* Note early continue */
/* Not available locally, queue a HTTP request */
g_hash_table_add (pull_data->requested_content, file_checksum);
enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE, NULL);
file_checksum = NULL; /* Transfer ownership */
}
g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1);
const guint m = g_variant_n_children (dirs_variant);
for (guint i = 0; i < m; i++)
{
const char *dirname = NULL;
g_autoptr(GVariant) tree_csum = NULL;
g_autoptr(GVariant) meta_csum = NULL;
g_variant_get_child (dirs_variant, i, "(&s@ay@ay)",
&dirname, &tree_csum, &meta_csum);
/* See comment above for files */
if (!ot_util_filename_validate (dirname, error))
return FALSE;
if (!pull_matches_subdir (pull_data, path, dirname, TRUE))
continue;
const guchar *tree_csum_bytes = ostree_checksum_bytes_peek_validate (tree_csum, error);
if (tree_csum_bytes == NULL)
return FALSE;
const guchar *meta_csum_bytes = ostree_checksum_bytes_peek_validate (meta_csum, error);
if (meta_csum_bytes == NULL)
return FALSE;
g_autofree char *subpath = g_strconcat (path, dirname, "/", NULL);
queue_scan_one_metadata_object_c (pull_data, tree_csum_bytes,
OSTREE_OBJECT_TYPE_DIR_TREE, subpath, recursion_depth + 1, NULL);
queue_scan_one_metadata_object_c (pull_data, meta_csum_bytes,
OSTREE_OBJECT_TYPE_DIR_META, subpath, recursion_depth + 1, NULL);
}
return TRUE;
}
/* Given a @ref, fetch its contents (should be a SHA256 ASCII string) */
static gboolean
fetch_ref_contents (OtPullData *pull_data,
const char *main_collection_id,
const OstreeCollectionRef *ref,
char **out_contents,
GCancellable *cancellable,
GError **error)
{
g_autofree char *ret_contents = NULL;
if (pull_data->remote_repo_local != NULL && ref->collection_id != NULL)
{
if (!ostree_repo_resolve_collection_ref (pull_data->remote_repo_local,
ref, FALSE,
OSTREE_REPO_RESOLVE_REV_EXT_NONE,
&ret_contents, cancellable, error))
return FALSE;
}
else if (pull_data->remote_repo_local != NULL)
{
if (!ostree_repo_resolve_rev_ext (pull_data->remote_repo_local,
ref->ref_name, FALSE,
OSTREE_REPO_RESOLVE_REV_EXT_NONE,
&ret_contents, error))
return FALSE;
}
else
{
g_autofree char *filename = NULL;
if (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, main_collection_id) == 0)
filename = g_build_filename ("refs", "heads", ref->ref_name, NULL);
else
filename = g_build_filename ("refs", "mirrors", ref->collection_id, ref->ref_name, NULL);
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
filename, pull_data->n_network_retries,
&ret_contents,
cancellable, error))
return FALSE;
g_strchomp (ret_contents);
}
g_assert (ret_contents);
if (!ostree_validate_checksum_string (ret_contents, error))
return glnx_prefix_error (error, "Fetching checksum for ref (%s, %s)",
ref->collection_id ?: "(empty)",
ref->ref_name);
ot_transfer_out_value (out_contents, &ret_contents);
return TRUE;
}
static gboolean
lookup_commit_checksum_and_collection_from_summary (OtPullData *pull_data,
const OstreeCollectionRef *ref,
char **out_checksum,
gsize *out_size,
char **out_collection_id,
GError **error)
{
g_autoptr(GVariant) additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
const gchar *main_collection_id;
if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
main_collection_id = NULL;
g_autoptr(GVariant) refs = NULL;
const gchar *resolved_collection_id = NULL;
if (ref->collection_id == NULL || g_strcmp0 (ref->collection_id, main_collection_id) == 0)
{
refs = g_variant_get_child_value (pull_data->summary, 0);
resolved_collection_id = main_collection_id;
}
else if (ref->collection_id != NULL)
{
g_autoptr(GVariant) collection_map = NULL;
collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP,
G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
if (collection_map != NULL)
refs = g_variant_lookup_value (collection_map, ref->collection_id, G_VARIANT_TYPE ("a(s(taya{sv}))"));
resolved_collection_id = ref->collection_id;
}
int i;
if (refs == NULL || !ot_variant_bsearch_str (refs, ref->ref_name, &i))
{
if (ref->collection_id != NULL)
return glnx_throw (error, "No such branch (%s, %s) in repository summary", ref->collection_id, ref->ref_name);
else
return glnx_throw (error, "No such branch '%s' in repository summary", ref->ref_name);
}
g_autoptr(GVariant) refdata = g_variant_get_child_value (refs, i);
g_autoptr(GVariant) reftargetdata = g_variant_get_child_value (refdata, 1);
guint64 commit_size;
g_autoptr(GVariant) commit_csum_v = NULL;
g_variant_get (reftargetdata, "(t@ay@a{sv})", &commit_size, &commit_csum_v, NULL);
if (resolved_collection_id != NULL &&
!ostree_validate_collection_id (resolved_collection_id, error))
return FALSE;
if (!ostree_validate_structureof_csum_v (commit_csum_v, error))
return FALSE;
*out_checksum = ostree_checksum_from_bytes_v (commit_csum_v);
*out_size = commit_size;
*out_collection_id = g_strdup (resolved_collection_id);
return TRUE;
}
static void
fetch_object_data_free (FetchObjectData *fetch_data)
{
g_variant_unref (fetch_data->object);
g_free (fetch_data->path);
if (fetch_data->requested_ref)
ostree_collection_ref_free (fetch_data->requested_ref);
g_free (fetch_data);
}
static void
content_fetch_on_write_complete (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
FetchObjectData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
OstreeObjectType objtype;
const char *expected_checksum;
g_autofree guchar *csum = NULL;
g_autofree char *checksum = NULL;
g_autofree char *checksum_obj = NULL;
if (!ostree_repo_write_content_finish ((OstreeRepo*)object, result,
&csum, error))
goto out;
checksum = ostree_checksum_from_bytes (csum);
ostree_object_name_deserialize (fetch_data->object, &expected_checksum, &objtype);
g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
checksum_obj = ostree_object_to_string (checksum, objtype);
g_debug ("write of %s complete", checksum_obj);
if (!_ostree_compare_object_checksum (objtype, expected_checksum, checksum, error))
goto out;
pull_data->n_fetched_content++;
/* Was this a delta fallback? */
if (g_hash_table_remove (pull_data->requested_fallback_content, expected_checksum))
pull_data->n_fetched_deltapart_fallbacks++;
out:
pull_data->n_outstanding_content_write_requests--;
/* No retries for local writes. */
check_outstanding_requests_handle_error (pull_data, &local_error);
fetch_object_data_free (fetch_data);
}
static void
content_fetch_on_complete (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
OstreeFetcher *fetcher = (OstreeFetcher *)object;
FetchObjectData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
GCancellable *cancellable = NULL;
guint64 length;
g_auto(GLnxTmpfile) tmpf = { 0, };
g_autoptr(GInputStream) tmpf_input = NULL;
g_autoptr(GFileInfo) file_info = NULL;
g_autoptr(GVariant) xattrs = NULL;
g_autoptr(GInputStream) file_in = NULL;
g_autoptr(GInputStream) object_input = NULL;
const char *checksum;
g_autofree char *checksum_obj = NULL;
OstreeObjectType objtype;
gboolean free_fetch_data = TRUE;
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
goto out;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
checksum_obj = ostree_object_to_string (checksum, objtype);
g_debug ("fetch of %s complete", checksum_obj);
const gboolean verifying_bareuseronly =
(pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY) > 0;
/* See comments where we set this variable; this is implementing
* the --trusted-http/OSTREE_REPO_PULL_FLAGS_TRUSTED_HTTP flags.
*/
if (pull_data->trusted_http_direct)
{
g_assert (!verifying_bareuseronly);
if (!_ostree_repo_commit_tmpf_final (pull_data->repo, checksum, objtype,
&tmpf, cancellable, error))
goto out;
pull_data->n_fetched_content++;
}
else
{
struct stat stbuf;
if (!glnx_fstat (tmpf.fd, &stbuf, error))
goto out;
/* Non-mirroring path */
tmpf_input = g_unix_input_stream_new (glnx_steal_fd (&tmpf.fd), TRUE);
/* If it appears corrupted, we'll delete it below */
if (!ostree_content_stream_parse (TRUE, tmpf_input, stbuf.st_size, FALSE,
&file_in, &file_info, &xattrs,
cancellable, error))
{
g_prefix_error (error, "Parsing %s: ", checksum_obj);
goto out;
}
if (verifying_bareuseronly)
{
if (!_ostree_validate_bareuseronly_mode_finfo (file_info, checksum, error))
goto out;
}
if (!ostree_raw_file_to_content_stream (file_in, file_info, xattrs,
&object_input, &length,
cancellable, error))
goto out;
pull_data->n_outstanding_content_write_requests++;
ostree_repo_write_content_async (pull_data->repo, checksum,
object_input, length,
cancellable,
content_fetch_on_write_complete, fetch_data);
free_fetch_data = FALSE;
}
out:
g_assert (pull_data->n_outstanding_content_fetches > 0);
pull_data->n_outstanding_content_fetches--;
if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
else
check_outstanding_requests_handle_error (pull_data, &local_error);
if (free_fetch_data)
g_clear_pointer (&fetch_data, fetch_object_data_free);
}
static void
on_metadata_written (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
FetchObjectData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
const char *expected_checksum;
OstreeObjectType objtype;
g_autofree char *checksum = NULL;
g_autofree guchar *csum = NULL;
g_autofree char *stringified_object = NULL;
if (!ostree_repo_write_metadata_finish ((OstreeRepo*)object, result,
&csum, error))
goto out;
checksum = ostree_checksum_from_bytes (csum);
ostree_object_name_deserialize (fetch_data->object, &expected_checksum, &objtype);
g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
stringified_object = ostree_object_to_string (checksum, objtype);
g_debug ("write of %s complete", stringified_object);
if (strcmp (checksum, expected_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted metadata object; checksum expected='%s' actual='%s'",
expected_checksum, checksum);
goto out;
}
queue_scan_one_metadata_object_c (pull_data, csum, objtype, fetch_data->path, 0, fetch_data->requested_ref);
out:
g_assert (pull_data->n_outstanding_metadata_write_requests > 0);
pull_data->n_outstanding_metadata_write_requests--;
fetch_object_data_free (fetch_data);
/* No need to retry local write operations. */
check_outstanding_requests_handle_error (pull_data, &local_error);
}
static void
meta_fetch_on_complete (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
OstreeFetcher *fetcher = (OstreeFetcher *)object;
FetchObjectData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GVariant) metadata = NULL;
g_auto(GLnxTmpfile) tmpf = { 0, };
const char *checksum;
g_autofree char *checksum_obj = NULL;
OstreeObjectType objtype;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
gboolean free_fetch_data = TRUE;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
checksum_obj = ostree_object_to_string (checksum, objtype);
g_debug ("fetch of %s%s complete", checksum_obj,
fetch_data->is_detached_meta ? " (detached)" : "");
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
if (fetch_data->is_detached_meta)
{
/* There isn't any detached metadata, just fetch the commit */
g_clear_error (&local_error);
/* Now that we've at least tried to fetch it, we can proceed to
* scan/fetch the commit object */
g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
if (!fetch_data->object_is_stored)
enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE, fetch_data->requested_ref);
else
queue_scan_one_metadata_object (pull_data, checksum, objtype, fetch_data->path, 0, fetch_data->requested_ref);
}
/* When traversing parents, do not fail on a missing commit.
* We may be pulling from a partial repository that ends in
* a dangling parent reference. */
else if (objtype == OSTREE_OBJECT_TYPE_COMMIT &&
pull_data->maxdepth != 0)
{
g_clear_error (&local_error);
/* If the remote repo supports tombstone commits, check if the commit was intentionally
deleted. */
if (pull_data->has_tombstone_commits)
{
enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT,
fetch_data->path, FALSE, FALSE, NULL);
}
}
}
goto out;
}
/* Tombstone commits are always empty, so skip all processing here */
if (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT)
goto out;
if (fetch_data->is_detached_meta)
{
if (!ot_variant_read_fd (tmpf.fd, 0, G_VARIANT_TYPE ("a{sv}"),
FALSE, &metadata, error))
goto out;
if (!ostree_repo_write_commit_detached_metadata (pull_data->repo, checksum, metadata,
pull_data->cancellable, error))
goto out;
g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), g_steal_pointer (&metadata));
if (!fetch_data->object_is_stored)
enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE, fetch_data->requested_ref);
else
queue_scan_one_metadata_object (pull_data, checksum, objtype, fetch_data->path, 0, fetch_data->requested_ref);
}
else
{
if (!ot_variant_read_fd (tmpf.fd, 0, ostree_metadata_variant_type (objtype),
FALSE, &metadata, error))
goto out;
/* Compute checksum and verify structure now. Note this is a recent change
* (Jan 2018) - we used to verify the checksum only when writing down
* below. But we want to do "structure" verification early on as well
* before the object is written even to the staging directory.
*/
if (!_ostree_verify_metadata_object (objtype, checksum, metadata, error))
goto out;
/* For commit objects, check the signature before writing to the repo,
* and also write the .commitpartial to say that we're still processing
* this commit.
*/
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
/* Do signature verification. `detached_data` may be NULL if no detached
* metadata was found during pull; that's handled by
* ostree_ostree_verify_unwritten_commit(). If we ever change the pull code to
* not always fetch detached metadata, this bit will have to learn how
* to look up from the disk state as well, or insert the on-disk
* metadata into this hash.
*/
GVariant *detached_data = g_hash_table_lookup (pull_data->fetched_detached_metadata, checksum);
if (!_verify_unwritten_commit (pull_data, checksum, metadata, detached_data,
fetch_data->requested_ref, pull_data->cancellable, error))
goto out;
if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
goto out;
}
/* Note that we now (Jan 2018) pass NULL for checksum, which means "don't
* verify checksum", since we just did it above. Related to this...now
* that we're doing all the verification here, one thing we could do later
* just `glnx_link_tmpfile_at()` into the repository, like the content
* fetch path does for trusted commits.
*/
ostree_repo_write_metadata_async (pull_data->repo, objtype, NULL, metadata,
pull_data->cancellable,
on_metadata_written, fetch_data);
pull_data->n_outstanding_metadata_write_requests++;
free_fetch_data = FALSE;
}
out:
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
pull_data->n_outstanding_metadata_fetches--;
if (local_error == NULL)
pull_data->n_fetched_metadata++;
if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
else
check_outstanding_requests_handle_error (pull_data, &local_error);
if (free_fetch_data)
g_clear_pointer (&fetch_data, fetch_object_data_free);
}
static void
fetch_static_delta_data_free (gpointer data)
{
FetchStaticDeltaData *fetch_data = data;
g_free (fetch_data->expected_checksum);
g_variant_unref (fetch_data->objects);
g_free (fetch_data->from_revision);
g_free (fetch_data->to_revision);
g_free (fetch_data);
}
static void
on_static_delta_written (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
FetchStaticDeltaData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
g_debug ("execute static delta part %s complete", fetch_data->expected_checksum);
if (!_ostree_static_delta_part_execute_finish (pull_data->repo, result, error))
goto out;
out:
g_assert (pull_data->n_outstanding_deltapart_write_requests > 0);
pull_data->n_outstanding_deltapart_write_requests--;
/* No need to retry on failure to write locally. */
check_outstanding_requests_handle_error (pull_data, &local_error);
/* Always free state */
fetch_static_delta_data_free (fetch_data);
}
static void
static_deltapart_fetch_on_complete (GObject *object,
GAsyncResult *result,
gpointer user_data)
{
OstreeFetcher *fetcher = (OstreeFetcher *)object;
FetchStaticDeltaData *fetch_data = user_data;
OtPullData *pull_data = fetch_data->pull_data;
g_auto(GLnxTmpfile) tmpf = { 0, };
g_autoptr(GInputStream) in = NULL;
g_autoptr(GVariant) part = NULL;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
gboolean free_fetch_data = TRUE;
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
goto out;
/* Transfer ownership of the fd */
in = g_unix_input_stream_new (glnx_steal_fd (&tmpf.fd), TRUE);
/* TODO - make async */
if (!_ostree_static_delta_part_open (in, NULL, 0, fetch_data->expected_checksum,
&part, pull_data->cancellable, error))
goto out;
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
part,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
pull_data->n_outstanding_deltapart_write_requests++;
free_fetch_data = FALSE;
out:
g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
pull_data->n_outstanding_deltapart_fetches--;
if (local_error == NULL)
pull_data->n_fetched_deltaparts++;
if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data));
else
check_outstanding_requests_handle_error (pull_data, &local_error);
if (free_fetch_data)
g_clear_pointer (&fetch_data, fetch_static_delta_data_free);
}
static gboolean
commitstate_is_partial (OtPullData *pull_data,
OstreeRepoCommitState commitstate)
{
return pull_data->legacy_transaction_resuming
|| (commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL) > 0;
}
#endif /* HAVE_LIBCURL_OR_LIBSOUP */
/* Reads the collection-id of a given remote from the repo
* configuration.
*/
static char *
get_real_remote_repo_collection_id (OstreeRepo *repo,
const gchar *remote_name)
{
/* remote_name == NULL can happen for pull-local */
if (!remote_name)
return NULL;
g_autofree gchar *remote_collection_id = NULL;
if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL,
&remote_collection_id, NULL) ||
(remote_collection_id == NULL) ||
(remote_collection_id[0] == '\0'))
return NULL;
return g_steal_pointer (&remote_collection_id);
}
#ifdef HAVE_LIBCURL_OR_LIBSOUP
/* Reads the collection-id of the remote repo. Where it will be read
* from depends on whether we pull from the "local" remote repo (the
* "file://" URL) or "remote" remote repo (likely the "http(s)://"
* URL).
*/
static char *
get_remote_repo_collection_id (OtPullData *pull_data)
{
if (pull_data->remote_repo_local != NULL)
{
const char *remote_collection_id =
ostree_repo_get_collection_id (pull_data->remote_repo_local);
if ((remote_collection_id == NULL) ||
(remote_collection_id[0] == '\0'))
return NULL;
return g_strdup (remote_collection_id);
}
return get_real_remote_repo_collection_id (pull_data->repo,
pull_data->remote_name);
}
#endif /* HAVE_LIBCURL_OR_LIBSOUP */
/* Check whether the given remote exists, has a `collection-id` key set, and it
* equals @collection_id. If so, return %TRUE. Otherwise, %FALSE. */
static gboolean
check_remote_matches_collection_id (OstreeRepo *repo,
const gchar *remote_name,
const gchar *collection_id)
{
g_autofree gchar *remote_collection_id = NULL;
remote_collection_id = get_real_remote_repo_collection_id (repo, remote_name);
if (remote_collection_id == NULL)
return FALSE;
return g_str_equal (remote_collection_id, collection_id);
}
/**
* ostree_repo_resolve_keyring_for_collection:
* @self: an #OstreeRepo
* @collection_id: the collection ID to look up a keyring for
* @cancellable: (nullable): a #GCancellable, or %NULL
* @error: return location for a #GError, or %NULL
*
* Find the GPG keyring for the given @collection_id, using the local
* configuration from the given #OstreeRepo. This will search the configured
* remotes for ones whose `collection-id` key matches @collection_id, and will
* return the first matching remote.
*
* If multiple remotes match and have different keyrings, a debug message will
* be emitted, and the first result will be returned. It is expected that the
* keyrings should match.
*
* If no match can be found, a %G_IO_ERROR_NOT_FOUND error will be returned.
*
* Returns: (transfer full): #OstreeRemote containing the GPG keyring for
* @collection_id
* Since: 2018.6
*/
OstreeRemote *
ostree_repo_resolve_keyring_for_collection (OstreeRepo *self,
const gchar *collection_id,
GCancellable *cancellable,
GError **error)
{
#ifndef OSTREE_DISABLE_GPGME
gsize i;
g_auto(GStrv) remotes = NULL;
g_autoptr(OstreeRemote) keyring_remote = NULL;
g_return_val_if_fail (OSTREE_IS_REPO (self), NULL);
g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL);
g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), NULL);
g_return_val_if_fail (error == NULL || *error == NULL, NULL);
/* Look through all the currently configured remotes for the given collection. */
remotes = ostree_repo_remote_list (self, NULL);
for (i = 0; remotes != NULL && remotes[i] != NULL; i++)
{
g_autoptr(GError) local_error = NULL;
if (!check_remote_matches_collection_id (self, remotes[i], collection_id))
continue;
if (keyring_remote == NULL)
{
g_debug ("%s: Found match for collection ‘%s’ in remote ‘%s’.",
G_STRFUNC, collection_id, remotes[i]);
keyring_remote = _ostree_repo_get_remote_inherited (self, remotes[i], &local_error);
if (keyring_remote == NULL)
{
g_debug ("%s: Error loading remote ‘%s’: %s",
G_STRFUNC, remotes[i], local_error->message);
continue;
}
if (g_strcmp0 (keyring_remote->keyring, "") == 0 ||
g_strcmp0 (keyring_remote->keyring, "/dev/null") == 0)
{
g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.",
G_STRFUNC, remotes[i]);
g_clear_object (&keyring_remote);
continue;
}
/* continue so we can catch duplicates */
}
else
{
g_debug ("%s: Duplicate keyring for collection ‘%s’ in remote ‘%s’."
"Keyring will be loaded from remote ‘%s’.",
G_STRFUNC, collection_id, remotes[i],
keyring_remote->name);
}
}
if (keyring_remote != NULL)
return g_steal_pointer (&keyring_remote);
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"No keyring found configured locally for collection ‘%s’",
collection_id);
return NULL;
}
#else
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"'%s': GPG feature is disabled in a build time",
__FUNCTION__);
return NULL;
#endif /* OSTREE_DISABLE_GPGME */
}
#ifdef HAVE_LIBCURL_OR_LIBSOUP
/* Look at a commit object, and determine whether there are
* more things to fetch.
*/
static gboolean
scan_commit_object (OtPullData *pull_data,
const char *checksum,
guint recursion_depth,
const OstreeCollectionRef *ref,
GCancellable *cancellable,
GError **error)
{
gpointer depthp;
gint depth;
if (g_hash_table_lookup_extended (pull_data->commit_to_depth, checksum,
NULL, &depthp))
{
depth = GPOINTER_TO_INT (depthp);
}
else
{
depth = pull_data->maxdepth;
g_hash_table_insert (pull_data->commit_to_depth, g_strdup (checksum),
GINT_TO_POINTER (depth));
}
#ifndef OSTREE_DISABLE_GPGME
/* See comment in process_verify_result() - we now gpg check before writing,
* but also ensure we've done it here if not already.
*/
if (pull_data->gpg_verify &&
!g_hash_table_contains (pull_data->verified_commits, checksum))
{
g_autoptr(OstreeGpgVerifyResult) result = NULL;
const char *keyring_remote = NULL;
if (ref != NULL)
keyring_remote = g_hash_table_lookup (pull_data->ref_keyring_map, ref);
if (keyring_remote == NULL)
keyring_remote = pull_data->remote_name;
result = ostree_repo_verify_commit_for_remote (pull_data->repo,
checksum,
keyring_remote,
cancellable,
error);
if (!_process_gpg_verify_result (pull_data, checksum, result, error))
return FALSE;
}
#endif /* OSTREE_DISABLE_GPGME */
if (pull_data->signapi_commit_verifiers &&
!g_hash_table_contains (pull_data->signapi_verified_commits, checksum))
{
g_autoptr(GError) last_verification_error = NULL;
gboolean found_any_signature = FALSE;
gboolean found_valid_signature = FALSE;
g_autofree char *success_message = NULL;
for (guint i = 0; i < pull_data->signapi_commit_verifiers->len; i++)
{
OstreeSign *sign = pull_data->signapi_commit_verifiers->pdata[i];
found_any_signature = TRUE;
/* Set return to true if any sign fit */
if (ostree_sign_commit_verify (sign,
pull_data->repo,
checksum,
&success_message,
cancellable,
last_verification_error ? NULL : &last_verification_error))
{
found_valid_signature = TRUE;
break;
}
}
if (!found_any_signature)
return glnx_throw (error, "No signatures found for commit %s", checksum);
if (!found_valid_signature)
{
g_assert (last_verification_error);
g_propagate_error (error, g_steal_pointer (&last_verification_error));
return glnx_prefix_error (error, "Can't verify commit %s", checksum);
}
g_assert (success_message);
g_hash_table_insert (pull_data->signapi_verified_commits, g_strdup (checksum), g_steal_pointer (&success_message));
}
/* If we found a legacy transaction flag, assume we have to scan.
* We always do a scan of dirtree objects; see
* https://github.com/ostreedev/ostree/issues/543
*/
OstreeRepoCommitState commitstate;
g_autoptr(GVariant) commit = NULL;
if (!ostree_repo_load_commit (pull_data->repo, checksum, &commit, &commitstate, error))
return FALSE;
/* If ref is non-NULL then the commit we fetched was requested through the
* branch, otherwise we requested a commit checksum without specifying a branch.
*/
g_autofree char *remote_collection_id = NULL;
remote_collection_id = get_remote_repo_collection_id (pull_data);
if (!_ostree_repo_verify_bindings (remote_collection_id,
(ref != NULL) ? ref->ref_name : NULL,
commit, error))
return glnx_prefix_error (error, "Commit %s", checksum);
guint64 new_ts = ostree_commit_get_timestamp (commit);
if (pull_data->timestamp_check)
{
/* We don't support timestamp checking while recursing right now */
g_assert (ref);
g_assert_cmpint (recursion_depth, ==, 0);
const char *orig_rev = NULL;
if (!g_hash_table_lookup_extended (pull_data->ref_original_commits,
ref, NULL, (void**)&orig_rev))
g_assert_not_reached ();
g_autoptr(GVariant) orig_commit = NULL;
if (orig_rev)
{
if (!ostree_repo_load_commit (pull_data->repo, orig_rev,
&orig_commit, NULL, error))
return glnx_prefix_error (error, "Reading %s for timestamp-check", ref->ref_name);
guint64 orig_ts = ostree_commit_get_timestamp (orig_commit);
if (!_ostree_compare_timestamps (orig_rev, orig_ts, checksum, new_ts, error))
return FALSE;
}
}
if (pull_data->timestamp_check_from_rev)
{
g_autoptr(GVariant) timestamp_commit = NULL;
if (!ostree_repo_load_commit (pull_data->repo, pull_data->timestamp_check_from_rev,
×tamp_commit, NULL, error))
return glnx_prefix_error (error, "Reading %s for timestamp-check-from-rev",
pull_data->timestamp_check_from_rev);
guint64 ts = ostree_commit_get_timestamp (timestamp_commit);
if (!_ostree_compare_timestamps (pull_data->timestamp_check_from_rev, ts, checksum, new_ts, error))
return FALSE;
}
/* If we found a legacy transaction flag, assume all commits are partial */
gboolean is_partial = commitstate_is_partial (pull_data, commitstate);
/* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
g_autoptr(GVariant) parent_csum = NULL;
const guchar *parent_csum_bytes = NULL;
g_variant_get_child (commit, 1, "@ay", &parent_csum);
if (g_variant_n_children (parent_csum) > 0)
{
parent_csum_bytes = ostree_checksum_bytes_peek_validate (parent_csum, error);
if (parent_csum_bytes == NULL)
return FALSE;
}
if (parent_csum_bytes != NULL && pull_data->maxdepth == -1)
{
queue_scan_one_metadata_object_c (pull_data, parent_csum_bytes,
OSTREE_OBJECT_TYPE_COMMIT, NULL,
recursion_depth + 1, NULL);
}
else if (parent_csum_bytes != NULL && depth > 0)
{
char parent_checksum[OSTREE_SHA256_STRING_LEN+1];
gpointer parent_depthp;
int parent_depth;
ostree_checksum_inplace_from_bytes (parent_csum_bytes, parent_checksum);
if (g_hash_table_lookup_extended (pull_data->commit_to_depth, parent_checksum,
NULL, &parent_depthp))
{
parent_depth = GPOINTER_TO_INT (parent_depthp);
}
else
{
parent_depth = depth - 1;
}
if (parent_depth >= 0)
{
g_hash_table_insert (pull_data->commit_to_depth, g_strdup (parent_checksum),
GINT_TO_POINTER (parent_depth));
queue_scan_one_metadata_object_c (pull_data, parent_csum_bytes,
OSTREE_OBJECT_TYPE_COMMIT,
NULL,
recursion_depth + 1,
NULL);
}
}
/* We only recurse to looking whether we need dirtree/dirmeta
* objects if the commit is partial, and we're not doing a
* commit-only fetch.
*/
if (is_partial && !pull_data->is_commit_only)
{
g_autoptr(GVariant) tree_contents_csum = NULL;
g_autoptr(GVariant) tree_meta_csum = NULL;
const guchar *tree_contents_csum_bytes;
const guchar *tree_meta_csum_bytes;
g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
tree_contents_csum_bytes = ostree_checksum_bytes_peek_validate (tree_contents_csum, error);
if (tree_contents_csum_bytes == NULL)
return FALSE;
tree_meta_csum_bytes = ostree_checksum_bytes_peek_validate (tree_meta_csum, error);
if (tree_meta_csum_bytes == NULL)
return FALSE;
queue_scan_one_metadata_object_c (pull_data, tree_contents_csum_bytes,
OSTREE_OBJECT_TYPE_DIR_TREE, "/", recursion_depth + 1, NULL);
queue_scan_one_metadata_object_c (pull_data, tree_meta_csum_bytes,
OSTREE_OBJECT_TYPE_DIR_META, NULL, recursion_depth + 1, NULL);
}
return TRUE;
}
static void
queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref)
{
guchar buf[OSTREE_SHA256_DIGEST_LEN];
ostree_checksum_inplace_to_bytes (csum, buf);
queue_scan_one_metadata_object_c (pull_data, buf, objtype, path, recursion_depth, ref);
}
static void
queue_scan_one_metadata_object_s (OtPullData *pull_data,
ScanObjectQueueData *scan_data)
{
g_queue_push_tail (&pull_data->scan_object_queue, scan_data);
ensure_idle_queued (pull_data);
}
static void
queue_scan_one_metadata_object_c (OtPullData *pull_data,
const guchar *csum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref)
{
ScanObjectQueueData *scan_data = g_new0 (ScanObjectQueueData, 1);
memcpy (scan_data->csum, csum, sizeof (scan_data->csum));
scan_data->objtype = objtype;
scan_data->path = g_strdup (path);
scan_data->recursion_depth = recursion_depth;
scan_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
queue_scan_one_metadata_object_s (pull_data, g_steal_pointer (&scan_data));
}
/* Called out of the main loop to look at metadata objects which can have
* further references (commit, dirtree). See also idle_worker() which drives
* execution of this function.
*/
static gboolean
scan_one_metadata_object (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
const char *path,
guint recursion_depth,
const OstreeCollectionRef *ref,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GVariant) object = ostree_object_name_serialize (checksum, objtype);
/* It may happen that we've already looked at this object (think shared
* dirtree subtrees), if that's the case, we're done */
if (g_hash_table_lookup (pull_data->scanned_metadata, object))
return TRUE;
gboolean is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL;
/* Determine if we already have the object */
gboolean is_stored;
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored,
cancellable, error))
return FALSE;
/* Are we pulling an object we don't have from a local repo? */
if (!is_stored && pull_data->remote_repo_local)
{
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
/* mark as partial to ensure we scan the commit below */
if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
return FALSE;
}
if (!_ostree_repo_import_object (pull_data->repo, pull_data->remote_repo_local,
objtype, checksum, pull_data->importflags,
cancellable, error))
return FALSE;
/* The import API will fetch both the commit and detached metadata, so
* add it to the hash to avoid re-fetching it below.
*/
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
pull_data->n_imported_metadata++;
is_stored = TRUE;
is_requested = TRUE;
}
/* Do we have any localcache repos? */
else if (!is_stored && pull_data->localcache_repos)
{
for (guint i = 0; i < pull_data->localcache_repos->len; i++)
{
OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i];
gboolean localcache_repo_has_obj;
if (!ostree_repo_has_object (refd_repo, objtype, checksum,
&localcache_repo_has_obj, cancellable, error))
return FALSE;
if (!localcache_repo_has_obj)
continue;
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
/* mark as partial to ensure we scan the commit below */
if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
return FALSE;
}
if (!_ostree_repo_import_object (pull_data->repo, refd_repo,
objtype, checksum, pull_data->importflags,
cancellable, error))
return FALSE;
/* See comment above */
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
is_stored = TRUE;
is_requested = TRUE;
pull_data->n_imported_metadata++;
break;
}
}
if (!is_stored && !is_requested)
{
gboolean do_fetch_detached;
g_hash_table_add (pull_data->requested_metadata, g_variant_ref (object));
do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
enqueue_one_object_request (pull_data, checksum, objtype, path, do_fetch_detached, FALSE, ref);
}
else if (is_stored && objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
/* Even though we already have the commit, we always try to (re)fetch the
* detached metadata before scanning it, in case new signatures appear.
* https://github.com/projectatomic/rpm-ostree/issues/630 */
if (!g_hash_table_contains (pull_data->fetched_detached_metadata, checksum))
enqueue_one_object_request (pull_data, checksum, objtype, path, TRUE, TRUE, ref);
else
{
if (!scan_commit_object (pull_data, checksum, recursion_depth, ref,
pull_data->cancellable, error))
return FALSE;
g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object));
pull_data->n_scanned_metadata++;
}
}
else if (is_stored && objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
{
if (!scan_dirtree_object (pull_data, checksum, path, recursion_depth,
pull_data->cancellable, error))
return FALSE;
g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object));
pull_data->n_scanned_metadata++;
}
return TRUE;
}
static void
enqueue_one_object_request_s (OtPullData *pull_data,
FetchObjectData *fetch_data)
{
const char *checksum;
OstreeObjectType objtype;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
/* Are too many requests are in flight? */
if (fetcher_queue_is_full (pull_data))
{
g_debug ("queuing fetch of %s.%s%s", checksum,
ostree_object_type_to_string (objtype),
fetch_data->is_detached_meta ? " (detached)" : "");
if (is_meta)
{
g_hash_table_insert (pull_data->pending_fetch_metadata, g_variant_ref (fetch_data->object), fetch_data);
}
else
{
g_hash_table_insert (pull_data->pending_fetch_content, g_strdup (checksum), fetch_data);
}
}
else
{
start_fetch (pull_data, fetch_data);
}
}
static void
enqueue_one_object_request (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
const char *path,
gboolean is_detached_meta,
gboolean object_is_stored,
const OstreeCollectionRef *ref)
{
FetchObjectData *fetch_data;
fetch_data = g_new0 (FetchObjectData, 1);
fetch_data->pull_data = pull_data;
fetch_data->object = ostree_object_name_serialize (checksum, objtype);
fetch_data->path = g_strdup (path);
fetch_data->is_detached_meta = is_detached_meta;
fetch_data->object_is_stored = object_is_stored;
fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
fetch_data->n_retries_remaining = pull_data->n_network_retries;
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
pull_data->n_requested_metadata++;
else
pull_data->n_requested_content++;
enqueue_one_object_request_s (pull_data, g_steal_pointer (&fetch_data));
}
static void
start_fetch (OtPullData *pull_data,
FetchObjectData *fetch)
{
g_autofree char *obj_subpath = NULL;
guint64 *expected_max_size_p;
guint64 expected_max_size;
const char *expected_checksum;
OstreeObjectType objtype;
GPtrArray *mirrorlist = NULL;
ostree_object_name_deserialize (fetch->object, &expected_checksum, &objtype);
g_debug ("starting fetch of %s.%s%s", expected_checksum,
ostree_object_type_to_string (objtype),
fetch->is_detached_meta ? " (detached)" : "");
gboolean is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (is_meta)
pull_data->n_outstanding_metadata_fetches++;
else
pull_data->n_outstanding_content_fetches++;
OstreeFetcherRequestFlags flags = 0;
/* Override the path if we're trying to fetch the .commitmeta file first */
if (fetch->is_detached_meta)
{
char buf[_OSTREE_LOOSE_PATH_MAX];
_ostree_loose_path (buf, expected_checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
obj_subpath = g_build_filename ("objects", buf, NULL);
mirrorlist = pull_data->meta_mirrorlist;
flags |= OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT;
}
else
{
obj_subpath = _ostree_get_relative_object_path (expected_checksum, objtype, TRUE);
mirrorlist = pull_data->content_mirrorlist;
}
/* We may have determined maximum sizes from the summary file content; if so,
* honor it. Otherwise, metadata has a baseline max size.
*/
expected_max_size_p = fetch->is_detached_meta ? NULL : g_hash_table_lookup (pull_data->expected_commit_sizes, expected_checksum);
if (expected_max_size_p)
expected_max_size = *expected_max_size_p;
else if (OSTREE_OBJECT_TYPE_IS_META (objtype))
expected_max_size = pull_data->max_metadata_size;
else
expected_max_size = 0;
if (!is_meta && pull_data->trusted_http_direct)
flags |= OSTREE_FETCHER_REQUEST_LINKABLE;
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
obj_subpath, flags, expected_max_size,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch);
}
/* Deprecated: code should load options from the `summary` file rather than
* downloading the remote’s `config` file, to save on network round trips. */
static gboolean
load_remote_repo_config (OtPullData *pull_data,
GKeyFile **out_keyfile,
GCancellable *cancellable,
GError **error)
{
g_autofree char *contents = NULL;
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
"config", pull_data->n_network_retries,
&contents, cancellable, error))
return FALSE;
g_autoptr(GKeyFile) ret_keyfile = g_key_file_new ();
if (!g_key_file_load_from_data (ret_keyfile, contents, strlen (contents),
0, error))
return glnx_prefix_error (error, "Parsing config");
ot_transfer_out_value (out_keyfile, &ret_keyfile);
return TRUE;
}
static gboolean
process_one_static_delta_fallback (OtPullData *pull_data,
gboolean delta_byteswap,
GVariant *fallback_object,
GCancellable *cancellable,
GError **error)
{
guint8 objtype_y;
g_autoptr(GVariant) csum_v = NULL;
guint64 compressed_size, uncompressed_size;
g_variant_get (fallback_object, "(y@aytt)",
&objtype_y, &csum_v, &compressed_size, &uncompressed_size);
if (!ostree_validate_structureof_objtype (objtype_y, error))
return FALSE;
if (!ostree_validate_structureof_csum_v (csum_v, error))
return FALSE;
compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
pull_data->n_total_delta_fallbacks += 1;
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
OstreeObjectType objtype = (OstreeObjectType)objtype_y;
g_autofree char *checksum = ostree_checksum_from_bytes_v (csum_v);
gboolean is_stored;
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
&is_stored,
cancellable, error))
return FALSE;
if (is_stored)
pull_data->fetched_deltapart_size += compressed_size;
if (pull_data->dry_run)
return TRUE; /* Note early return */
if (!is_stored)
{
/* The delta compiler never did this, there's no reason to support it */
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
return glnx_throw (error, "Found metadata object as fallback: %s.%s", checksum,
ostree_object_type_to_string (objtype));
else
{
if (!g_hash_table_lookup (pull_data->requested_content, checksum))
{
/* Mark this as requested, like we do in the non-delta path */
g_hash_table_add (pull_data->requested_content, checksum);
/* But also record it's a delta fallback object, so we can account
* for it as logically part of the delta fetch.
*/
g_hash_table_add (pull_data->requested_fallback_content, g_strdup (checksum));
enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, NULL, FALSE, FALSE, NULL);
checksum = NULL; /* We transferred ownership to the requested_content hash */
}
}
}
return TRUE;
}
static void
enqueue_one_static_delta_part_request_s (OtPullData *pull_data,
FetchStaticDeltaData *fetch_data)
{
if (fetcher_queue_is_full (pull_data))
{
g_debug ("queuing fetch of static delta %s-%s part %u",
fetch_data->from_revision ?: "empty",
fetch_data->to_revision, fetch_data->i);
g_hash_table_add (pull_data->pending_fetch_deltaparts, fetch_data);
}
else
{
start_fetch_deltapart (pull_data, fetch_data);
}
}
static void
start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch)
{
g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i);
pull_data->n_outstanding_deltapart_fetches++;
g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, 0, fetch->size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch);
}
static gboolean
process_one_static_delta (OtPullData *pull_data,
const char *from_revision,
const char *to_revision,
GVariant *delta_superblock,
const OstreeCollectionRef *ref,
GCancellable *cancellable,
GError **error)
{
gboolean delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
g_autoptr(GVariant) metadata = g_variant_get_child_value (delta_superblock, 0);
g_autoptr(GVariant) headers = g_variant_get_child_value (delta_superblock, 6);
g_autoptr(GVariant) fallback_objects = g_variant_get_child_value (delta_superblock, 7);
/* Gather free space so we can do a check below */
struct statvfs stvfsbuf;
if (TEMP_FAILURE_RETRY (fstatvfs (pull_data->repo->repo_dir_fd, &stvfsbuf)) < 0)
return glnx_throw_errno_prefix (error, "fstatvfs");
/* First process the fallbacks */
guint n = g_variant_n_children (fallback_objects);
for (guint i = 0; i < n; i++)
{
g_autoptr(GVariant) fallback_object =
g_variant_get_child_value (fallback_objects, i);
if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
fallback_object,
cancellable, error))
return FALSE;
}
/* Write the to-commit object */
if (!pull_data->dry_run)
{
g_autoptr(GVariant) to_csum_v = g_variant_get_child_value (delta_superblock, 3);
if (!ostree_validate_structureof_csum_v (to_csum_v, error))
return FALSE;
g_autofree char *to_checksum = ostree_checksum_from_bytes_v (to_csum_v);
gboolean have_to_commit;
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
&have_to_commit, cancellable, error))
return FALSE;
if (!have_to_commit)
{
g_autoptr(GVariant) to_commit = g_variant_get_child_value (delta_superblock, 4);
g_autofree char *detached_path = _ostree_get_relative_static_delta_path (from_revision, to_revision, "commitmeta");
g_autoptr(GVariant) detached_data = g_variant_lookup_value (metadata, detached_path, G_VARIANT_TYPE("a{sv}"));
if (!_verify_unwritten_commit (pull_data, to_revision, to_commit, detached_data,
ref, cancellable, error))
return FALSE;
if (detached_data && !ostree_repo_write_commit_detached_metadata (pull_data->repo,
to_revision,
detached_data,
cancellable,
error))
return FALSE;
FetchObjectData *fetch_data = g_new0 (FetchObjectData, 1);
fetch_data->pull_data = pull_data;
fetch_data->object = ostree_object_name_serialize (to_checksum, OSTREE_OBJECT_TYPE_COMMIT);
fetch_data->is_detached_meta = FALSE;
fetch_data->object_is_stored = FALSE;
fetch_data->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
fetch_data->n_retries_remaining = pull_data->n_network_retries;
ostree_repo_write_metadata_async (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
to_commit,
pull_data->cancellable,
on_metadata_written, fetch_data);
pull_data->n_outstanding_metadata_write_requests++;
}
}
n = g_variant_n_children (headers);
pull_data->n_total_deltaparts += n;
for (guint i = 0; i < n; i++)
{
gboolean have_all = FALSE;
g_autoptr(GVariant) header = g_variant_get_child_value (headers, i);
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GBytes) inline_part_bytes = NULL;
guint32 version;
guint64 size, usize;
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
version = maybe_swap_endian_u32 (delta_byteswap, version);
size = maybe_swap_endian_u64 (delta_byteswap, size);
usize = maybe_swap_endian_u64 (delta_byteswap, usize);
if (version > OSTREE_DELTAPART_VERSION)
return glnx_throw (error, "Delta part has too new version %u", version);
const guchar *csum = ostree_checksum_bytes_peek_validate (csum_v, error);
if (!csum)
return FALSE;
if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
objects,
&have_all,
cancellable, error))
return FALSE;
pull_data->total_deltapart_size += size;
pull_data->total_deltapart_usize += usize;
if (have_all)
{
g_debug ("Have all objects from static delta %s-%s part %u",
from_revision ?: "empty", to_revision,
i);
pull_data->fetched_deltapart_size += size;
pull_data->n_fetched_deltaparts++;
continue;
}
g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
{ g_autoptr(GVariant) part_datav =
g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)"));
if (part_datav)
inline_part_bytes = g_variant_get_data_as_bytes (part_datav);
}
if (pull_data->dry_run)
continue;
FetchStaticDeltaData *fetch_data = g_new0 (FetchStaticDeltaData, 1);
fetch_data->from_revision = g_strdup (from_revision);
fetch_data->to_revision = g_strdup (to_revision);
fetch_data->pull_data = pull_data;
fetch_data->objects = g_variant_ref (objects);
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
fetch_data->size = size;
fetch_data->i = i;
fetch_data->n_retries_remaining = pull_data->n_network_retries;
if (inline_part_bytes != NULL)
{
g_autoptr(GInputStream) memin = g_memory_input_stream_new_from_bytes (inline_part_bytes);
g_autoptr(GVariant) inline_delta_part = NULL;
/* For inline parts we are relying on per-commit GPG, so don't bother checksumming. */
if (!_ostree_static_delta_part_open (memin, inline_part_bytes,
OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM,
NULL, &inline_delta_part,
cancellable, error))
{
fetch_static_delta_data_free (fetch_data);
return FALSE;
}
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
inline_delta_part,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
pull_data->n_outstanding_deltapart_write_requests++;
}
else
{
enqueue_one_static_delta_part_request_s (pull_data, g_steal_pointer (&fetch_data));
}
}
/* The free space check is here since at this point we've parsed the delta not
* only the total size of the parts, but also whether or not we already have
* them. TODO: Ideally this free space check would be above, but we'd have to
* walk everything twice and keep track of state.
*/
const guint64 delta_required_blocks = (pull_data->total_deltapart_usize / stvfsbuf.f_bsize);
if (delta_required_blocks > stvfsbuf.f_bfree)
{
g_autofree char *formatted_required = g_format_size (pull_data->total_deltapart_usize);
g_autofree char *formatted_avail = g_format_size (((guint64)stvfsbuf.f_bsize) * stvfsbuf.f_bfree);
return glnx_throw (error, "Delta requires %s free space, but only %s available",
formatted_required, formatted_avail);
}
return TRUE;
}
/*
* DELTA_SEARCH_RESULT_UNCHANGED:
* We already have the commit.
*
* DELTA_SEARCH_RESULT_NO_MATCH:
* No deltas were found.
*
* DELTA_SEARCH_RESULT_FROM:
* A regular delta was found, and the "from" revision will be
* set in `from_revision`.
*
* DELTA_SEARCH_RESULT_SCRATCH:
* There is a %NULL → @to_revision delta, also known as
* a "from scratch" delta.
*/
typedef struct {
enum {
DELTA_SEARCH_RESULT_UNCHANGED,
DELTA_SEARCH_RESULT_NO_MATCH,
DELTA_SEARCH_RESULT_FROM,
DELTA_SEARCH_RESULT_SCRATCH,
} result;
char from_revision[OSTREE_SHA256_STRING_LEN+1];
} DeltaSearchResult;
/* Loop over the static delta data we got from the summary,
* and find the a delta path (if available) that goes to @to_revision.
* See the enum in `DeltaSearchResult` for available result types.
*/
static gboolean
get_best_static_delta_start_for (OtPullData *pull_data,
const char *to_revision,
DeltaSearchResult *out_result,
GCancellable *cancellable,
GError **error)
{
/* Array<char*> of possible from checksums */
g_autoptr(GPtrArray) candidates = g_ptr_array_new_with_free_func (g_free);
const char *newest_candidate = NULL;
guint64 newest_candidate_timestamp = 0;
g_assert (pull_data->summary_deltas_checksums != NULL);
out_result->result = DELTA_SEARCH_RESULT_NO_MATCH;
out_result->from_revision[0] = '\0';
/* First, do we already have this commit completely downloaded? */
gboolean have_to_rev;
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT,
to_revision, &have_to_rev,
cancellable, error))
return FALSE;
if (have_to_rev)
{
OstreeRepoCommitState to_rev_state;
if (!ostree_repo_load_commit (pull_data->repo, to_revision,
NULL, &to_rev_state, error))
return FALSE;
if (!(to_rev_state & OSTREE_REPO_COMMIT_STATE_PARTIAL))
{
/* We already have this commit, we're done! */
out_result->result = DELTA_SEARCH_RESULT_UNCHANGED;
return TRUE; /* Early return */
}
}
/* Loop over all deltas known from the summary file,
* finding ones which go to to_revision */
GLNX_HASH_TABLE_FOREACH (pull_data->summary_deltas_checksums, const char*, delta_name)
{
g_autofree char *cur_from_rev = NULL;
g_autofree char *cur_to_rev = NULL;
/* Gracefully handle corrupted (or malicious) summary files */
if (!_ostree_parse_delta_name (delta_name, &cur_from_rev, &cur_to_rev, error))
return FALSE;
/* Is this the checksum we want? */
if (strcmp (cur_to_rev, to_revision) != 0)
continue;
if (cur_from_rev)
{
g_ptr_array_add (candidates, g_steal_pointer (&cur_from_rev));
}
else
{
/* We note that we have a _SCRATCH delta here, but we'll prefer using
* "from" deltas (obviously, they'll be smaller) where possible if we
* find one below.
*/
out_result->result = DELTA_SEARCH_RESULT_SCRATCH;
}
}
/* Loop over our candidates, find the newest one */
for (guint i = 0; i < candidates->len; i++)
{
const char *candidate = candidates->pdata[i];
guint64 candidate_ts = 0;
g_autoptr(GVariant) commit = NULL;
OstreeRepoCommitState state;
gboolean have_candidate;
/* Do we have this commit at all? If not, skip it */
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT,
candidate, &have_candidate,
NULL, error))
return FALSE;
if (!have_candidate)
continue;
/* Load it */
if (!ostree_repo_load_commit (pull_data->repo, candidate,
&commit, &state, error))
return FALSE;
/* Ignore partial commits, we can't use them */
if (state & OSTREE_REPO_COMMIT_STATE_PARTIAL)
continue;
/* Is it newer? */
candidate_ts = ostree_commit_get_timestamp (commit);
if (newest_candidate == NULL ||
candidate_ts > newest_candidate_timestamp)
{
newest_candidate = candidate;
newest_candidate_timestamp = candidate_ts;
}
}
if (newest_candidate)
{
out_result->result = DELTA_SEARCH_RESULT_FROM;
memcpy (out_result->from_revision, newest_candidate, OSTREE_SHA256_STRING_LEN+1);
}
return TRUE;
}
static void
fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data)
{
g_free (fetch_data->from_revision);
g_free (fetch_data->to_revision);
if (fetch_data->requested_ref)
ostree_collection_ref_free (fetch_data->requested_ref);
g_free (fetch_data);
}
static void
set_required_deltas_error (GError **error,
const char *from_revision,
const char *to_revision)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Static deltas required, but none found for %s to %s",
from_revision, to_revision);
}
static void
on_superblock_fetched (GObject *src,
GAsyncResult *res,
gpointer data)
{
FetchDeltaSuperData *fetch_data = data;
OtPullData *pull_data = fetch_data->pull_data;
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
g_autoptr(GBytes) delta_superblock_data = NULL;
const char *from_revision = fetch_data->from_revision;
const char *to_revision = fetch_data->to_revision;
if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
res,
&delta_superblock_data,
error))
{
if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
goto out;
g_clear_error (&local_error);
if (pull_data->require_static_deltas)
{
set_required_deltas_error (error, from_revision, to_revision);
goto out;
}
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, fetch_data->requested_ref);
}
else
{
g_autoptr(GVariant) delta_superblock = NULL;
g_autofree gchar *delta = g_strconcat (from_revision ?: "", from_revision ? "-" : "", to_revision, NULL);
const guchar *expected_summary_digest = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
guint8 actual_summary_digest[OSTREE_SHA256_DIGEST_LEN];
ot_checksum_bytes (delta_superblock_data, actual_summary_digest);
#ifndef OSTREE_DISABLE_GPGME
/* At this point we've GPG verified the data, so in theory
* could trust that they provided the right data, but let's
* make this a hard error.
*/
if (pull_data->gpg_verify_summary && !expected_summary_digest)
{
g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
"GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
#endif /* OSTREE_DISABLE_GPGME */
if (expected_summary_digest && memcmp (expected_summary_digest, actual_summary_digest, sizeof (actual_summary_digest)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
goto out;
}
delta_superblock = g_variant_ref_sink (g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
delta_superblock_data, FALSE));
g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock));
if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock, fetch_data->requested_ref,
pull_data->cancellable, error))
goto out;
}
out:
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
pull_data->n_outstanding_metadata_fetches--;
if (local_error == NULL)
pull_data->n_fetched_metadata++;
if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
enqueue_one_static_delta_superblock_request_s (pull_data, g_steal_pointer (&fetch_data));
else
check_outstanding_requests_handle_error (pull_data, &local_error);
g_clear_pointer (&fetch_data, fetch_delta_super_data_free);
}
static void
start_fetch_delta_superblock (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data)
{
g_autofree char *delta_name =
_ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision,
fetch_data->to_revision);
_ostree_fetcher_request_to_membuf (pull_data->fetcher,
pull_data->content_mirrorlist,
delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
OSTREE_MAX_METADATA_SIZE,
0, pull_data->cancellable,
on_superblock_fetched,
g_steal_pointer (&fetch_data));
pull_data->n_outstanding_metadata_fetches++;
pull_data->n_requested_metadata++;
}
static void
enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data,
FetchDeltaSuperData *fetch_data)
{
if (fetcher_queue_is_full (pull_data))
{
g_debug ("queuing fetch of static delta superblock %s-%s",
fetch_data->from_revision ?: "empty",
fetch_data->to_revision);
g_hash_table_add (pull_data->pending_fetch_delta_superblocks,
g_steal_pointer (&fetch_data));
}
else
{
start_fetch_delta_superblock (pull_data, g_steal_pointer (&fetch_data));
}
}
/* Start a request for a static delta */
static void
enqueue_one_static_delta_superblock_request (OtPullData *pull_data,
const char *from_revision,
const char *to_revision,
const OstreeCollectionRef *ref)
{
FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1);
fdata->pull_data = pull_data;
fdata->from_revision = g_strdup (from_revision);
fdata->to_revision = g_strdup (to_revision);
fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
fdata->n_retries_remaining = pull_data->n_network_retries;
enqueue_one_static_delta_superblock_request_s (pull_data, g_steal_pointer (&fdata));
}
static gboolean
validate_variant_is_csum (GVariant *csum,
GError **error)
{
if (!g_variant_is_of_type (csum, G_VARIANT_TYPE ("ay")))
return glnx_throw (error, "Invalid checksum variant of type '%s', expected 'ay'",
g_variant_get_type_string (csum));
return ostree_validate_structureof_csum_v (csum, error);
}
static gboolean
_ostree_repo_verify_summary (OstreeRepo *self,
const char *name,
gboolean gpg_verify_summary,
GPtrArray *signapi_summary_verifiers,
GBytes *summary,
GBytes *signatures,
GCancellable *cancellable,
GError **error)
{
if (gpg_verify_summary)
{
if (summary == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)");
return FALSE;
}
if (signatures == NULL)
{
g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
"GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
return FALSE;
}
/* Verify any summary signatures. */
if (summary != NULL && signatures != NULL)
{
g_autoptr(OstreeGpgVerifyResult) result = NULL;
result = ostree_repo_verify_summary (self,
name,
summary,
signatures,
cancellable,
error);
if (!ostree_gpg_verify_result_require_valid_signature (result, error))
return FALSE;
}
}
if (signapi_summary_verifiers)
{
if (summary == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Signature verification enabled, but no summary found (check that the configured URL in remote config is correct)");
return FALSE;
}
if (signatures == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Signature verification enabled, but no summary signatures found (use sign-verify-summary=false in remote config to disable)");
return FALSE;
}
/* Verify any summary signatures. */
if (summary != NULL && signatures != NULL)
{
g_autoptr(GVariant) sig_variant = NULL;
sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
signatures, FALSE);
if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, NULL, error))
return FALSE;
}
}
return TRUE;
}
static gboolean
_ostree_repo_load_cache_summary_file (OstreeRepo *self,
const char *filename,
const char *extension,
GBytes **out_data,
GCancellable *cancellable,
GError **error)
{
const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension);
glnx_autofd int fd = -1;
g_autoptr(GBytes) data = NULL;
*out_data = NULL;
if (self->cache_dir_fd == -1)
return TRUE;
fd = openat (self->cache_dir_fd, file, O_CLOEXEC | O_RDONLY);
if (fd < 0)
{
if (errno == ENOENT)
return TRUE;
return glnx_throw_errno_prefix (error, "openat(%s)", file);
}
data = ot_fd_readall_or_mmap (fd, 0, error);
if (!data)
return FALSE;
*out_data =g_steal_pointer (&data);
return TRUE;
}
/* Load the summary from the cache if the provided .sig file is the same as the
cached version. */
static gboolean
_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self,
const char *remote,
GBytes *summary_sig,
GBytes **out_summary,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GBytes) old_sig_contents = NULL;
*out_summary = NULL;
if (!_ostree_repo_load_cache_summary_file (self, remote, ".sig",
&old_sig_contents,
cancellable, error))
return FALSE;
if (old_sig_contents != NULL &&
g_bytes_compare (old_sig_contents, summary_sig) == 0)
{
g_autoptr(GBytes) summary_data = NULL;
if (!_ostree_repo_load_cache_summary_file (self, remote, NULL,
&summary_data,
cancellable, error))
return FALSE;
if (summary_data == NULL)
{
/* Cached signature without cached summary, remove the signature */
const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig");
(void) unlinkat (self->cache_dir_fd, summary_cache_sig_file, 0);
}
else
*out_summary = g_steal_pointer (&summary_data);
}
return TRUE;
}
static gboolean
_ostree_repo_save_cache_summary_file (OstreeRepo *self,
const char *filename,
const char *extension,
GBytes *data,
GCancellable *cancellable,
GError **error)
{
const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension);
glnx_autofd int fd = -1;
if (self->cache_dir_fd == -1)
return TRUE;
if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, DEFAULT_DIRECTORY_MODE, cancellable, error))
return FALSE;
if (!glnx_file_replace_contents_at (self->cache_dir_fd,
file,
g_bytes_get_data (data, NULL),
g_bytes_get_size (data),
self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW,
cancellable, error))
return FALSE;
return TRUE;
}
/* Replace the current summary+signature with new versions */
static gboolean
_ostree_repo_cache_summary (OstreeRepo *self,
const char *remote,
GBytes *summary,
GBytes *summary_sig,
GCancellable *cancellable,
GError **error)
{
if (!_ostree_repo_save_cache_summary_file (self, remote, NULL,
summary, cancellable, error))
return FALSE;
if (!_ostree_repo_save_cache_summary_file (self, remote, ".sig",
summary_sig, cancellable, error))
return FALSE;
return TRUE;
}
static OstreeFetcher *
_ostree_repo_remote_new_fetcher (OstreeRepo *self,
const char *remote_name,
gboolean gzip,
GVariant *extra_headers,
const char *append_user_agent,
OstreeFetcherSecurityState *out_state,
GError **error)
{
OstreeFetcher *fetcher = NULL;
OstreeFetcherConfigFlags fetcher_flags = 0;
gboolean tls_permissive = FALSE;
OstreeFetcherSecurityState ret_state = OSTREE_FETCHER_SECURITY_STATE_TLS;
gboolean success = FALSE;
g_return_val_if_fail (OSTREE_IS_REPO (self), NULL);
g_return_val_if_fail (remote_name != NULL, NULL);
if (!ostree_repo_get_remote_boolean_option (self, remote_name,
"tls-permissive", FALSE,
&tls_permissive, error))
goto out;
if (tls_permissive)
{
fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE;
ret_state = OSTREE_FETCHER_SECURITY_STATE_INSECURE;
}
if (gzip)
fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
{ gboolean http2_default = TRUE;
#ifndef BUILDOPT_HTTP2
http2_default = FALSE;
#endif
gboolean http2;
if (!ostree_repo_get_remote_boolean_option (self, remote_name,
"http2", http2_default,
&http2, error))
goto out;
if (!http2)
fetcher_flags |= OSTREE_FETCHER_FLAGS_DISABLE_HTTP2;
}
fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
{
g_autofree char *tls_client_cert_path = NULL;
g_autofree char *tls_client_key_path = NULL;
if (!ostree_repo_get_remote_option (self, remote_name,
"tls-client-cert-path", NULL,
&tls_client_cert_path, error))
goto out;
if (!ostree_repo_get_remote_option (self, remote_name,
"tls-client-key-path", NULL,
&tls_client_key_path, error))
goto out;
if ((tls_client_cert_path != NULL) != (tls_client_key_path != NULL))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote \"%s\" must specify both "
"\"tls-client-cert-path\" and \"tls-client-key-path\"",
remote_name);
goto out;
}
else if (tls_client_cert_path != NULL)
{
_ostree_fetcher_set_client_cert (fetcher, tls_client_cert_path, tls_client_key_path);
}
}
{
g_autofree char *tls_ca_path = NULL;
if (!ostree_repo_get_remote_option (self, remote_name,
"tls-ca-path", NULL,
&tls_ca_path, error))
goto out;
if (tls_ca_path != NULL)
{
_ostree_fetcher_set_tls_database (fetcher, tls_ca_path);
/* Don't change if it's already _INSECURE */
if (ret_state == OSTREE_FETCHER_SECURITY_STATE_TLS)
ret_state = OSTREE_FETCHER_SECURITY_STATE_CA_PINNED;
}
}
{
g_autofree char *http_proxy = NULL;
if (!ostree_repo_get_remote_option (self, remote_name,
"proxy", NULL,
&http_proxy, error))
goto out;
if (http_proxy != NULL && http_proxy[0] != '\0')
_ostree_fetcher_set_proxy (fetcher, http_proxy);
}
if (!_ostree_repo_remote_name_is_file (remote_name))
{
g_autofree char *cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name);
/* TODO; port away from this; a bit hard since both libsoup and libcurl
* expect a file. Doing ot_fdrel_to_gfile() works for now though.
*/
GFile*repo_path = ostree_repo_get_path (self);
g_autofree char *jar_path =
g_build_filename (gs_file_get_path_cached (repo_path), cookie_file, NULL);
if (g_file_test (jar_path, G_FILE_TEST_IS_REGULAR))
_ostree_fetcher_set_cookie_jar (fetcher, jar_path);
}
if (extra_headers)
_ostree_fetcher_set_extra_headers (fetcher, extra_headers);
if (append_user_agent)
_ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent);
success = TRUE;
out:
if (!success)
g_clear_object (&fetcher);
if (out_state)
*out_state = ret_state;
return fetcher;
}
static gboolean
_ostree_preload_metadata_file (OstreeRepo *self,
OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
gboolean is_metalink,
guint n_network_retries,
GBytes **out_bytes,
GCancellable *cancellable,
GError **error)
{
if (is_metalink)
{
GError *local_error = NULL;
/* the metalink uri is buried in the mirrorlist as the first (and only)
* element */
g_autoptr(OstreeMetalink) metalink =
_ostree_metalink_new (fetcher, filename,
OSTREE_MAX_METADATA_SIZE,
mirrorlist->pdata[0], n_network_retries);
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
cancellable, &local_error);
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
g_clear_error (&local_error);
*out_bytes = NULL;
}
else if (local_error != NULL)
{
g_propagate_error (error, local_error);
return FALSE;
}
return TRUE;
}
else
{
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
n_network_retries,
out_bytes, OSTREE_MAX_METADATA_SIZE,
cancellable, error);
}
}
static gboolean
fetch_mirrorlist (OstreeFetcher *fetcher,
const char *mirrorlist_url,
guint n_network_retries,
GPtrArray **out_mirrorlist,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GPtrArray) ret_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_autoptr(OstreeFetcherURI) mirrorlist = _ostree_fetcher_uri_parse (mirrorlist_url, error);
if (!mirrorlist)
return FALSE;
g_autofree char *contents = NULL;
if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, n_network_retries,
&contents, cancellable, error))
return glnx_prefix_error (error, "While fetching mirrorlist '%s'",
mirrorlist_url);
/* go through each mirror in mirrorlist and do a quick sanity check that it
* works so that we don't waste the fetcher's time when it goes through them
* */
g_auto(GStrv) lines = g_strsplit (contents, "\n", -1);
g_debug ("Scanning mirrorlist from '%s'", mirrorlist_url);
for (char **iter = lines; iter && *iter; iter++)
{
const char *mirror_uri_str = *iter;
g_autoptr(OstreeFetcherURI) mirror_uri = NULL;
g_autofree char *scheme = NULL;
/* let's be nice and support empty lines and comments */
if (*mirror_uri_str == '\0' || *mirror_uri_str == '#')
continue;
mirror_uri = _ostree_fetcher_uri_parse (mirror_uri_str, NULL);
if (!mirror_uri)
{
g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str);
continue;
}
scheme = _ostree_fetcher_uri_get_scheme (mirror_uri);
if (!(g_str_equal (scheme, "http") || (g_str_equal (scheme, "https"))))
{
/* let's not support mirrorlists that contain non-http based URIs for
* now (e.g. local URIs) -- we need to think about if and how we want
* to support this since we set up things differently depending on
* whether we're pulling locally or not */
g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str);
continue;
}
/* We keep sanity checking until we hit a working mirror; there's no need
* to waste resources checking the remaining ones. At the same time,
* guaranteeing that the first mirror in the list works saves the fetcher
* time from always iterating through a few bad first mirrors. */
if (ret_mirrorlist->len == 0)
{
GError *local_error = NULL;
g_autoptr(OstreeFetcherURI) config_uri = _ostree_fetcher_uri_new_subpath (mirror_uri, "config");
if (fetch_uri_contents_utf8_sync (fetcher, config_uri, n_network_retries,
NULL, cancellable, &local_error))
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
else
{
g_debug ("Failed to fetch config from mirror '%s': %s",
mirror_uri_str, local_error->message);
g_clear_error (&local_error);
}
}
else
{
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
}
}
if (ret_mirrorlist->len == 0)
return glnx_throw (error, "No valid mirrors were found in mirrorlist '%s'",
mirrorlist_url);
*out_mirrorlist = g_steal_pointer (&ret_mirrorlist);
return TRUE;
}
static gboolean
compute_effective_mirrorlist (OstreeRepo *self,
const char *remote_name_or_baseurl,
const char *url_override,
OstreeFetcher *fetcher,
guint n_network_retries,
GPtrArray **out_mirrorlist,
GCancellable *cancellable,
GError **error)
{
g_autofree char *baseurl = NULL;
if (url_override != NULL)
baseurl = g_strdup (url_override);
else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error))
return FALSE;
if (g_str_has_prefix (baseurl, "mirrorlist="))
{
if (!fetch_mirrorlist (fetcher,
baseurl + strlen ("mirrorlist="),
n_network_retries,
out_mirrorlist,
cancellable, error))
return FALSE;
}
else
{
g_autoptr(OstreeFetcherURI) baseuri = _ostree_fetcher_uri_parse (baseurl, error);
if (!baseuri)
return FALSE;
*out_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (*out_mirrorlist, g_steal_pointer (&baseuri));
}
return TRUE;
}
/* Create the fetcher by unioning options from the remote config, plus
* any options specific to this pull (such as extra headers).
*/
static gboolean
reinitialize_fetcher (OtPullData *pull_data, const char *remote_name,
GError **error)
{
g_clear_object (&pull_data->fetcher);
pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, FALSE,
pull_data->extra_headers,
pull_data->append_user_agent,
&pull_data->fetcher_security_state,
error);
if (pull_data->fetcher == NULL)
return FALSE;
return TRUE;
}
/*
* initiate_request:
* @ref: Optional ref name and collection ID
* @to_revision: Target commit revision we want to fetch
*
* Start a request for either a ref or a commit. In the
* ref case, we know both the name and the target commit.
*
* This function primarily handles the semantics around
* `disable_static_deltas` and `require_static_deltas`.
*/
static gboolean
initiate_request (OtPullData *pull_data,
const OstreeCollectionRef *ref,
const char *to_revision,
GError **error)
{
g_autofree char *delta_from_revision = NULL;
/* Are deltas disabled? OK, just start an object fetch and be done */
if (pull_data->disable_static_deltas)
{
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
return TRUE;
}
/* If doing a delta from a ref, look up the from-revision, since we need it
* on most paths below. */
if (ref != NULL)
{
g_autofree char *refspec = NULL;
if (pull_data->remote_name != NULL)
refspec = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name);
if (!ostree_repo_resolve_rev (pull_data->repo,
refspec ?: ref->ref_name, TRUE,
&delta_from_revision, error))
return FALSE;
}
/* If we have a summary, we can use the newer logic */
if (pull_data->summary)
{
DeltaSearchResult deltares;
/* Look for a delta to @to_revision in the summary data */
if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares,
pull_data->cancellable, error))
return FALSE;
switch (deltares.result)
{
case DELTA_SEARCH_RESULT_NO_MATCH:
{
if (pull_data->require_static_deltas) /* No deltas found; are they required? */
{
set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision);
return FALSE;
}
else /* No deltas, fall back to object fetches. */
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
}
break;
case DELTA_SEARCH_RESULT_FROM:
enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref);
break;
case DELTA_SEARCH_RESULT_SCRATCH:
{
/* If a from-scratch delta is available, we don’t want to use it if
* the ref already exists locally, since we are likely only a few
* commits out of date; so doing an object pull is likely more
* bandwidth efficient. */
if (delta_from_revision != NULL)
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
else
enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref);
}
break;
case DELTA_SEARCH_RESULT_UNCHANGED:
{
/* If we already have the commit, here things get a little special; we've historically
* fetched detached metadata, so let's keep doing that. But in the --require-static-deltas
* path, we don't, under the assumption the user wants as little network traffic as
* possible.
*/
if (pull_data->require_static_deltas)
break;
else
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
}
}
}
else if (ref != NULL)
{
/* Are we doing a delta via a ref? In that case we can fall back to the older
* logic of just using the current tip of the ref as a delta FROM source. */
/* Determine whether the from revision we have is partial; this
* can happen if e.g. one uses `ostree pull --commit-metadata-only`.
* This mirrors the logic in get_best_static_delta_start_for().
*/
if (delta_from_revision)
{
OstreeRepoCommitState from_commitstate;
if (!ostree_repo_load_commit (pull_data->repo, delta_from_revision, NULL,
&from_commitstate, error))
return FALSE;
/* Was it partial? Then we can't use it. */
if (commitstate_is_partial (pull_data, from_commitstate))
g_clear_pointer (&delta_from_revision, g_free);
}
/* If the current ref is the same, we don't do a delta request, just a
* scan. Otherise, use the previous commit if available, or a scratch
* delta.
*/
if (delta_from_revision && g_str_equal (delta_from_revision, to_revision))
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
else
enqueue_one_static_delta_superblock_request (pull_data, delta_from_revision ?: NULL, to_revision, ref);
}
else
{
/* Legacy path without a summary file - let's try a scratch delta, if that
* doesn't work, it'll drop down to object requests.
*/
enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, NULL);
}
return TRUE;
}
/* ------------------------------------------------------------------------------------------
* Below is the libsoup-invariant API; these should match
* the stub functions in the #else clause
* ------------------------------------------------------------------------------------------
*/
/**
* ostree_repo_pull_with_options:
* @self: Repo
* @remote_name_or_baseurl: Name of remote or file:// url
* @options: A GVariant a{sv} with an extensible set of flags.
* @progress: (allow-none): Progress
* @cancellable: Cancellable
* @error: Error
*
* Like ostree_repo_pull(), but supports an extensible set of flags.
* The following are currently defined:
*
* * `refs` (`as`): Array of string refs
* * `collection-refs` (`a(sss)`): Array of (collection ID, ref name, checksum) tuples to pull;
* mutually exclusive with `refs` and `override-commit-ids`. Checksums may be the empty
* string to pull the latest commit for that ref
* * `flags` (`i`): An instance of #OstreeRepoPullFlags
* * `subdir` (`s`): Pull just this subdirectory
* * `subdirs` (`as`): Pull just these subdirectories
* * `override-remote-name` (`s`): If local, add this remote to refspec
* * `gpg-verify` (`b`): GPG verify commits
* * `gpg-verify-summary` (`b`): GPG verify summary
* * `disable-sign-verify` (`b`): Disable signapi verification of commits
* * `disable-sign-verify-summary` (`b`): Disable signapi verification of the summary
* * `depth` (`i`): How far in the history to traverse; default is 0, -1 means infinite
* * `per-object-fsync` (`b`): Perform disk writes more slowly, avoiding a single large I/O sync
* * `disable-static-deltas` (`b`): Do not use static deltas
* * `require-static-deltas` (`b`): Require static deltas
* * `override-commit-ids` (`as`): Array of specific commit IDs to fetch for refs
* * `timestamp-check` (`b`): Verify commit timestamps are newer than current (when pulling via ref); Since: 2017.11
* * `timestamp-check-from-rev` (`s`): Verify that all fetched commit timestamps are newer than timestamp of given rev; Since: 2020.4
* * `metadata-size-restriction` (`t`): Restrict metadata objects to a maximum number of bytes; 0 to disable. Since: 2018.9
* * `dry-run` (`b`): Only print information on what will be downloaded (requires static deltas)
* * `override-url` (`s`): Fetch objects from this URL if remote specifies no metalink in options
* * `inherit-transaction` (`b`): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction.
* * `http-headers` (`a(ss)`): Additional headers to add to all HTTP requests
* * `update-frequency` (`u`): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
* * `localcache-repos` (`as`): File paths for local repos to use as caches when doing remote fetches
* * `append-user-agent` (`s`): Additional string to append to the user agent
* * `n-network-retries` (`u`): Number of times to retry each download on receiving
* a transient network error, such as a socket timeout; default is 5, 0
* means return errors without retrying. Since: 2018.6
* * `ref-keyring-map` (`a(sss)`): Array of (collection ID, ref name, keyring
* remote name) tuples specifying which remote's keyring should be used when
* doing GPG verification of each collection-ref. This is useful to prevent a
* remote from serving malicious updates to refs which did not originate from
* it. This can be a subset or superset of the refs being pulled; any ref
* not being pulled will be ignored and any ref without a keyring remote
* will be verified with the keyring of the remote being pulled from.
* Since: 2019.2
* * `summary-bytes` (`ay'): Contents of the `summary` file to use. If this is
* specified, `summary-sig-bytes` must also be specified. This is
* useful if doing multiple pull operations in a transaction, using
* ostree_repo_remote_fetch_summary_with_options() beforehand to download
* the `summary` and `summary.sig` once for the entire transaction. If not
* specified, the `summary` will be downloaded from the remote. Since: 2020.5
* * `summary-sig-bytes` (`ay`): Contents of the `summary.sig` file. If this
* is specified, `summary-bytes` must also be specified. Since: 2020.5
*/
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
const char *remote_name_or_baseurl,
GVariant *options,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GBytes) bytes_summary = NULL;
g_autofree char *metalink_url_str = NULL;
g_autoptr(GHashTable) requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */
g_autoptr(GHashTable) commits_to_fetch = NULL;
g_autofree char *remote_mode_str = NULL;
g_autoptr(OstreeMetalink) metalink = NULL;
OtPullData pull_data_real = { 0, };
OtPullData *pull_data = &pull_data_real;
GKeyFile *remote_config = NULL;
char **configured_branches = NULL;
guint64 bytes_transferred;
guint64 end_time;
guint update_frequency = 0;
OstreeRepoPullFlags flags = 0;
const char *dir_to_pull = NULL;
g_autofree char **dirs_to_pull = NULL;
g_autofree char **refs_to_fetch = NULL;
g_autoptr(GVariantIter) collection_refs_iter = NULL;
g_autofree char **override_commit_ids = NULL;
g_autoptr(GSource) update_timeout = NULL;
gboolean opt_per_object_fsync = FALSE;
gboolean opt_gpg_verify_set = FALSE;
gboolean opt_gpg_verify_summary_set = FALSE;
gboolean opt_collection_refs_set = FALSE;
gboolean opt_n_network_retries_set = FALSE;
gboolean opt_ref_keyring_map_set = FALSE;
gboolean disable_sign_verify = FALSE;
gboolean disable_sign_verify_summary = FALSE;
const char *main_collection_id = NULL;
const char *url_override = NULL;
gboolean inherit_transaction = FALSE;
g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */
gsize i;
g_autofree char **opt_localcache_repos = NULL;
g_autoptr(GVariantIter) ref_keyring_map_iter = NULL;
g_autoptr(GVariant) summary_bytes_v = NULL;
g_autoptr(GVariant) summary_sig_bytes_v = NULL;
/* If refs or collection-refs has exactly one value, this will point to that
* value, otherwise NULL. Used for logging.
*/
const char *the_ref_to_fetch = NULL;
OstreeRepoTransactionStats tstats = { 0, };
/* Default */
pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE;
if (options)
{
int flags_i = OSTREE_REPO_PULL_FLAGS_NONE;
(void) g_variant_lookup (options, "refs", "^a&s", &refs_to_fetch);
opt_collection_refs_set =
g_variant_lookup (options, "collection-refs", "a(sss)", &collection_refs_iter);
(void) g_variant_lookup (options, "flags", "i", &flags_i);
/* Reduce risk of issues if enum happens to be 64 bit for some reason */
flags = flags_i;
(void) g_variant_lookup (options, "subdir", "&s", &dir_to_pull);
(void) g_variant_lookup (options, "subdirs", "^a&s", &dirs_to_pull);
(void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_refspec_name);
opt_gpg_verify_set =
g_variant_lookup (options, "gpg-verify", "b", &pull_data->gpg_verify);
opt_gpg_verify_summary_set =
g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary);
g_variant_lookup (options, "disable-sign-verify", "b", &disable_sign_verify);
g_variant_lookup (options, "disable-sign-verify-summary", "b", &disable_sign_verify_summary);
(void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth);
(void) g_variant_lookup (options, "disable-static-deltas", "b", &pull_data->disable_static_deltas);
(void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas);
(void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
(void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
(void) g_variant_lookup (options, "per-object-fsync", "b", &opt_per_object_fsync);
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
(void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
(void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
(void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
(void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos);
(void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check);
(void) g_variant_lookup (options, "timestamp-check-from-rev", "s", &pull_data->timestamp_check_from_rev);
(void) g_variant_lookup (options, "max-metadata-size", "t", &pull_data->max_metadata_size);
(void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent);
opt_n_network_retries_set =
g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries);
opt_ref_keyring_map_set =
g_variant_lookup (options, "ref-keyring-map", "a(sss)", &ref_keyring_map_iter);
(void) g_variant_lookup (options, "summary-bytes", "@ay", &summary_bytes_v);
(void) g_variant_lookup (options, "summary-sig-bytes", "@ay", &summary_sig_bytes_v);
if (pull_data->remote_refspec_name != NULL)
pull_data->remote_name = g_strdup (pull_data->remote_refspec_name);
}
#ifdef OSTREE_DISABLE_GPGME
/* Explicitly fail here if gpg verification is requested and we have no GPG support */
if (pull_data->gpg_verify || pull_data->gpg_verify_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"'%s': GPG feature is disabled at build time",
__FUNCTION__);
goto out;
}
#endif
g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
g_return_val_if_fail (!pull_data->timestamp_check || pull_data->maxdepth == 0, FALSE);
g_return_val_if_fail (!opt_collection_refs_set ||
(refs_to_fetch == NULL && override_commit_ids == NULL), FALSE);
if (refs_to_fetch && override_commit_ids)
g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE);
if (dir_to_pull)
g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
for (i = 0; dirs_to_pull != NULL && dirs_to_pull[i] != NULL; i++)
g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE);
g_return_val_if_fail (!(pull_data->disable_static_deltas && pull_data->require_static_deltas), FALSE);
/* We only do dry runs with static deltas, because we don't really have any
* in-advance information for bare fetches.
*/
g_return_val_if_fail (!pull_data->dry_run || pull_data->require_static_deltas, FALSE);
/* summary-bytes and summary-sig-bytes must both be specified, or neither be
* specified, so we know they’re consistent */
g_return_val_if_fail ((summary_bytes_v == NULL) == (summary_sig_bytes_v == NULL), FALSE);
pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
/* See our processing of OSTREE_REPO_PULL_FLAGS_UNTRUSTED below */
if ((flags & OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES) > 0)
pull_data->importflags |= _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY;
pull_data->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
if (error)
pull_data->async_error = &pull_data->cached_async_error;
else
pull_data->async_error = NULL;
/* Note we're using the thread default (or global) context here, so it may outlive the
* OtPullData object if there's another ref on it. Thus, always detach/destroy sources
* local to the `ostree_repo_pull*` operation rather than trying to transfer ownership. */
pull_data->main_context = g_main_context_ref_thread_default ();
pull_data->flags = flags;
/* TODO: Avoid mutating the repo object */
if (opt_per_object_fsync)
self->per_object_fsync = TRUE;
if (!opt_n_network_retries_set)
pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES;
pull_data->repo = self;
pull_data->progress = progress;
pull_data->expected_commit_sizes = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free,
(GDestroyNotify)g_free);
pull_data->commit_to_depth = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free,
NULL);
pull_data->summary_deltas_checksums = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free,
(GDestroyNotify)g_free);
pull_data->ref_original_commits = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal,
(GDestroyNotify)NULL,
(GDestroyNotify)g_free);
pull_data->verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->signapi_verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->ref_keyring_map = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal,
(GDestroyNotify)ostree_collection_ref_free, (GDestroyNotify)g_free);
pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref, NULL);
pull_data->fetched_detached_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, (GDestroyNotify)variant_or_null_unref);
pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->requested_fallback_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref, NULL);
pull_data->pending_fetch_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free,
(GDestroyNotify)fetch_object_data_free);
pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref,
(GDestroyNotify)fetch_object_data_free);
pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL);
pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
if (opt_localcache_repos && *opt_localcache_repos)
{
pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref);
for (char **it = opt_localcache_repos; it && *it; it++)
{
const char *localcache_path = *it;
g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path);
g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file);
if (!ostree_repo_open (cacherepo, cancellable, error))
goto out;
g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo));
}
}
if (dir_to_pull != NULL || dirs_to_pull != NULL)
{
pull_data->dirs = g_ptr_array_new_with_free_func (g_free);
if (dir_to_pull != NULL)
g_ptr_array_add (pull_data->dirs, g_strdup (dir_to_pull));
if (dirs_to_pull != NULL)
{
for (i = 0; dirs_to_pull[i] != NULL; i++)
g_ptr_array_add (pull_data->dirs, g_strdup (dirs_to_pull[i]));
}
}
g_queue_init (&pull_data->scan_object_queue);
pull_data->start_time = g_get_monotonic_time ();
if (_ostree_repo_remote_name_is_file (remote_name_or_baseurl))
{
/* For compatibility with pull-local, don't gpg verify local
* pulls by default.
*/
if ((pull_data->gpg_verify ||
pull_data->gpg_verify_summary
) &&
pull_data->remote_name == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Must specify remote name to enable gpg verification");
goto out;
}
}
else
{
g_autofree char *unconfigured_state = NULL;
g_free (pull_data->remote_name);
pull_data->remote_name = g_strdup (remote_name_or_baseurl);
/* Fetch GPG verification settings from remote if it wasn't already
* explicitly set in the options. */
if (!opt_gpg_verify_set)
if (!ostree_repo_remote_get_gpg_verify (self, pull_data->remote_name,
&pull_data->gpg_verify, error))
goto out;
if (!opt_gpg_verify_summary_set)
if (!ostree_repo_remote_get_gpg_verify_summary (self, pull_data->remote_name,
&pull_data->gpg_verify_summary, error))
goto out;
/* NOTE: If changing this, see the matching implementation in
* ostree-sysroot-upgrader.c
*/
if (!ostree_repo_get_remote_option (self, pull_data->remote_name,
"unconfigured-state", NULL,
&unconfigured_state,
error))
goto out;
if (unconfigured_state)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"remote unconfigured-state: %s", unconfigured_state);
goto out;
}
}
if (pull_data->remote_name && !(disable_sign_verify && disable_sign_verify_summary))
{
if (!_signapi_init_for_remote (pull_data->repo, pull_data->remote_name,
&pull_data->signapi_commit_verifiers,
&pull_data->signapi_summary_verifiers,
error))
return FALSE;
}
pull_data->phase = OSTREE_PULL_PHASE_FETCHING_REFS;
if (!reinitialize_fetcher (pull_data, remote_name_or_baseurl, error))
goto out;
pull_data->tmpdir_dfd = pull_data->repo->tmp_dir_fd;
requested_refs_to_fetch = g_hash_table_new_full (ostree_collection_ref_hash,
ostree_collection_ref_equal,
(GDestroyNotify) ostree_collection_ref_free,
g_free);
commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
if (!ostree_repo_get_remote_option (self,
remote_name_or_baseurl, "metalink",
NULL, &metalink_url_str, error))
goto out;
if (!metalink_url_str)
{
if (!compute_effective_mirrorlist (self, remote_name_or_baseurl,
url_override,
pull_data->fetcher,
pull_data->n_network_retries,
&pull_data->meta_mirrorlist,
cancellable, error))
goto out;
}
else
{
g_autoptr(GBytes) summary_bytes = NULL;
g_autoptr(OstreeFetcherURI) metalink_uri = _ostree_fetcher_uri_parse (metalink_url_str, error);
g_autoptr(OstreeFetcherURI) target_uri = NULL;
if (!metalink_uri)
goto out;
/* FIXME: Use summary_bytes_v/summary_sig_bytes_v to avoid unnecessary
* re-downloads here. Would require additional support for caching the
* metalink file or mirror list. */
metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
OSTREE_MAX_METADATA_SIZE, metalink_uri,
pull_data->n_network_retries);
if (! _ostree_metalink_request_sync (metalink,
&target_uri,
&summary_bytes,
cancellable,
error))
goto out;
/* XXX: would be interesting to implement metalink as another source of
* mirrors here since we use it as such anyway (rather than the "usual"
* use case of metalink, which is only for a single target filename) */
{
g_autofree char *path = _ostree_fetcher_uri_get_path (target_uri);
g_autofree char *basepath = g_path_get_dirname (path);
g_autoptr(OstreeFetcherURI) new_target_uri = _ostree_fetcher_uri_new_path (target_uri, basepath);
pull_data->meta_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, g_steal_pointer (&new_target_uri));
}
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
summary_bytes, FALSE);
}
{
g_autofree char *contenturl = NULL;
if (metalink_url_str == NULL && url_override != NULL)
contenturl = g_strdup (url_override);
else if (!ostree_repo_get_remote_option (self, remote_name_or_baseurl,
"contenturl", NULL,
&contenturl, error))
goto out;
if (contenturl == NULL)
{
pull_data->content_mirrorlist =
g_ptr_array_ref (pull_data->meta_mirrorlist);
}
else
{
if (!compute_effective_mirrorlist (self, remote_name_or_baseurl,
contenturl,
pull_data->fetcher,
pull_data->n_network_retries,
&pull_data->content_mirrorlist,
cancellable, error))
goto out;
}
}
/* FIXME: Do we want an analogue of this which supports collection IDs? */
if (!ostree_repo_get_remote_list_option (self,
remote_name_or_baseurl, "branches",
&configured_branches, error))
goto out;
/* TODO reindent later */
{ OstreeFetcherURI *first_uri = pull_data->meta_mirrorlist->pdata[0];
g_autofree char *first_scheme = _ostree_fetcher_uri_get_scheme (first_uri);
/* NB: we don't support local mirrors in mirrorlists, so if this passes, it
* means that we're not using mirrorlists (see also fetch_mirrorlist())
* Also, we explicitly disable the "local repo" path if static deltas
* were explicitly requested to be required; this is going to happen
* most often for testing deltas without setting up a HTTP server.
*/
if (g_str_equal (first_scheme, "file") && !pull_data->require_static_deltas)
{
g_autofree char *path = _ostree_fetcher_uri_get_path (first_uri);
g_autoptr(GFile) remote_repo_path = g_file_new_for_path (path);
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
goto out;
}
}
/* Change some option defaults if we're actually pulling from a local
* (filesystem accessible) repo.
*/
if (pull_data->remote_repo_local)
{
/* For local pulls, default to disabling static deltas so that the
* exact object files are copied.
*/
if (!pull_data->require_static_deltas)
pull_data->disable_static_deltas = TRUE;
/* Note the inversion here; PULL_FLAGS_UNTRUSTED is converted to
* IMPORT_FLAGS_TRUSTED only if it's unset (and just for local repos).
*/
if ((flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) == 0)
pull_data->importflags |= _OSTREE_REPO_IMPORT_FLAGS_TRUSTED;
/* Shouldn't be referenced in this path, but just in case. See below
* for more information.
*/
pull_data->trusted_http_direct = FALSE;
}
else
{
/* For non-local repos, we require the TRUSTED_HTTP pull flag to map to
* the TRUSTED object import flag. In practice we don't do object imports
* for HTTP, but it's easiest to use one set of flags between HTTP and
* local imports.
*/
if (flags & OSTREE_REPO_PULL_FLAGS_TRUSTED_HTTP)
pull_data->importflags |= _OSTREE_REPO_IMPORT_FLAGS_TRUSTED;
const gboolean verifying_bareuseronly =
(pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY) > 0;
/* If we're mirroring and writing into an archive repo, and both checksum and
* bareuseronly are turned off, we can directly copy the content rather than
* paying the cost of exploding it, checksumming, and re-gzip.
*/
const gboolean mirroring_into_archive =
pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE;
const gboolean import_trusted = !verifying_bareuseronly &&
(pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_TRUSTED) > 0;
pull_data->trusted_http_direct = mirroring_into_archive && import_trusted;
}
/* We can't use static deltas if pulling into an archive repo. */
if (self->mode == OSTREE_REPO_MODE_ARCHIVE)
{
if (pull_data->require_static_deltas)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Can't use static deltas in an archive repo");
goto out;
}
pull_data->disable_static_deltas = TRUE;
}
/* It's not efficient to use static deltas if all we want is the commit
* metadata. */
if (pull_data->is_commit_only)
pull_data->disable_static_deltas = TRUE;
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
{
g_autoptr(GBytes) bytes_sig = NULL;
gsize n;
g_autoptr(GVariant) refs = NULL;
g_autoptr(GVariant) deltas = NULL;
g_autoptr(GVariant) additional_metadata = NULL;
gboolean summary_from_cache = FALSE;
gboolean remote_mode_loaded = FALSE;
gboolean tombstone_commits = FALSE;
if (summary_sig_bytes_v)
{
/* Must both be specified */
g_assert (summary_bytes_v);
bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v);
bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v);
if (!bytes_sig || !bytes_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"summary-bytes or summary-sig-bytes set to invalid value");
goto out;
}
g_debug ("Loaded %s summary from options", remote_name_or_baseurl);
}
if (!bytes_sig)
{
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
pull_data->n_network_retries,
&bytes_sig,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
}
if (bytes_sig &&
!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_if_same_sig (self,
remote_name_or_baseurl,
bytes_sig,
&bytes_summary,
cancellable,
error))
goto out;
if (bytes_summary && !summary_bytes_v)
{
g_debug ("Loaded %s summary from cache", remote_name_or_baseurl);
summary_from_cache = TRUE;
}
if (!pull_data->summary && !bytes_summary)
{
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
pull_data->n_network_retries,
&bytes_summary,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_summary && pull_data->gpg_verify_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
#endif /* OSTREE_DISABLE_GPGME */
if (!bytes_summary && pull_data->require_static_deltas)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"Fetch configured to require static deltas, but no summary found");
goto out;
}
#ifndef OSTREE_DISABLE_GPGME
if (!bytes_sig && pull_data->gpg_verify_summary)
{
g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
"GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)");
goto out;
}
if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
{
g_autoptr(OstreeGpgVerifyResult) result = NULL;
g_autoptr(GError) temp_error = NULL;
result = ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, &temp_error);
if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
pull_data->n_network_retries,
&bytes_summary,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
g_autoptr(OstreeGpgVerifyResult) retry =
ostree_repo_verify_summary (self, pull_data->remote_name,
bytes_summary, bytes_sig,
cancellable, error);
if (!ostree_gpg_verify_result_require_valid_signature (retry, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
#endif /* OSTREE_DISABLE_GPGME */
if (pull_data->signapi_summary_verifiers)
{
if (!bytes_sig && pull_data->signapi_summary_verifiers)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)");
goto out;
}
if (bytes_summary && bytes_sig)
{
g_autoptr(GVariant) signatures = NULL;
g_autoptr(GError) temp_error = NULL;
signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
bytes_sig, FALSE);
g_assert (pull_data->signapi_summary_verifiers);
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Remote %s cached summary invalid and "
"OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified",
pull_data->remote_name);
goto out;
}
else
g_debug ("Remote %s cached summary invalid, pulling new version",
pull_data->remote_name);
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
pull_data->n_network_retries,
&bytes_summary,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error))
goto out;
}
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
goto out;
}
}
}
}
if (bytes_summary)
{
pull_data->summary_data = g_bytes_ref (bytes_summary);
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
if (!g_variant_is_normal_form (pull_data->summary))
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Not normal form");
goto out;
}
if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Doesn't match variant type '%s'",
(char *)OSTREE_SUMMARY_GVARIANT_FORMAT);
goto out;
}
if (bytes_sig)
pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
}
if (!summary_from_cache && bytes_summary && bytes_sig)
{
if (!pull_data->remote_repo_local &&
!_ostree_repo_cache_summary (self,
remote_name_or_baseurl,
bytes_summary,
bytes_sig,
cancellable,
error))
goto out;
}
if (pull_data->summary)
{
additional_metadata = g_variant_get_child_value (pull_data->summary, 1);
if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id))
main_collection_id = NULL;
else if (!ostree_validate_collection_id (main_collection_id, error))
goto out;
refs = g_variant_get_child_value (pull_data->summary, 0);
for (i = 0, n = g_variant_n_children (refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (main_collection_id, refname), NULL);
}
}
g_autoptr(GVariant) collection_map = NULL;
collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}"));
if (collection_map != NULL)
{
GVariantIter collection_map_iter;
const char *collection_id;
g_autoptr(GVariant) collection_refs = NULL;
g_variant_iter_init (&collection_map_iter, collection_map);
while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs))
{
if (!ostree_validate_collection_id (collection_id, error))
goto out;
for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++)
{
const char *refname;
g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i);
g_variant_get_child (ref, 0, "&s", &refname);
if (!ostree_validate_rev (refname, error))
goto out;
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set)
{
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (collection_id, refname), NULL);
}
}
}
}
deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
n = deltas ? g_variant_n_children (deltas) : 0;
for (i = 0; i < n; i++)
{
const char *delta;
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
g_variant_get_child (ref, 0, "&s", &delta);
g_variant_get_child (ref, 1, "v", &csum_v);
if (!validate_variant_is_csum (csum_v, error))
goto out;
guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32);
g_hash_table_insert (pull_data->summary_deltas_checksums,
g_strdup (delta),
csum_data);
}
}
if (pull_data->summary &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) &&
g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits))
{
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
pull_data->has_tombstone_commits = tombstone_commits;
remote_mode_loaded = TRUE;
}
else if (pull_data->remote_repo_local == NULL)
{
/* Fall-back path which loads the necessary config from the remote’s
* `config` file. Doing so is deprecated since it means an
* additional round trip to the remote for each pull. No need to do
* it for local pulls. */
if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
goto out;
if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
&remote_mode_str, error))
goto out;
if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
goto out;
if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "tombstone-commits", FALSE,
&pull_data->has_tombstone_commits, error))
goto out;
remote_mode_loaded = TRUE;
}
if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Can't pull from archives with mode \"%s\"",
remote_mode_str);
goto out;
}
}
if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
{
if (!bytes_summary)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
goto out;
}
}
else if (opt_collection_refs_set)
{
const gchar *collection_id, *ref_name, *checksum;
while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
{
if (!ostree_validate_rev (ref_name, error))
goto out;
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (collection_id, ref_name),
(*checksum != '\0') ? g_strdup (checksum) : NULL);
}
}
else if (refs_to_fetch != NULL)
{
char **strviter = refs_to_fetch;
char **commitid_strviter = override_commit_ids ?: NULL;
while (*strviter)
{
const char *branch = *strviter;
if (ostree_validate_checksum_string (branch, NULL))
{
char *key = g_strdup (branch);
g_hash_table_add (commits_to_fetch, key);
}
else
{
if (!ostree_validate_rev (branch, error))
goto out;
char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (NULL, branch), commitid);
}
strviter++;
if (commitid_strviter)
commitid_strviter++;
}
}
else
{
char **branches_iter;
branches_iter = configured_branches;
if (!(branches_iter && *branches_iter))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"No configured branches for remote %s", remote_name_or_baseurl);
goto out;
}
for (;branches_iter && *branches_iter; branches_iter++)
{
const char *branch = *branches_iter;
g_hash_table_insert (requested_refs_to_fetch,
ostree_collection_ref_new (NULL, branch), NULL);
}
}
/* Resolve the checksum for each ref. This has to be done into a new hash table,
* since we can’t modify the keys of @requested_refs_to_fetch while iterating
* over it, and we need to ensure the collection IDs are resolved too. */
updated_requested_refs_to_fetch = g_hash_table_new_full (ostree_collection_ref_hash,
ostree_collection_ref_equal,
(GDestroyNotify) ostree_collection_ref_free,
g_free);
GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref,
const char*, override_commitid)
{
g_autofree char *checksum = NULL;
g_autoptr(OstreeCollectionRef) ref_with_collection = NULL;
/* Support specifying "" for an override commitid */
if (override_commitid && *override_commitid)
{
ref_with_collection = ostree_collection_ref_dup (ref);
checksum = g_strdup (override_commitid);
}
else
{
if (pull_data->summary)
{
gsize commit_size = 0;
guint64 *malloced_size;
g_autofree gchar *collection_id = NULL;
if (!lookup_commit_checksum_and_collection_from_summary (pull_data, ref, &checksum, &commit_size, &collection_id, error))
goto out;
ref_with_collection = ostree_collection_ref_new (collection_id, ref->ref_name);
malloced_size = g_new0 (guint64, 1);
*malloced_size = commit_size;
g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (checksum), malloced_size);
}
else
{
if (!fetch_ref_contents (pull_data, main_collection_id, ref, &checksum, cancellable, error))
goto out;
ref_with_collection = ostree_collection_ref_dup (ref);
}
}
/* If we have timestamp checking enabled, find the current value of
* the ref, and store its timestamp in the hash map, to check later.
*/
if (pull_data->timestamp_check)
{
g_autofree char *from_rev = NULL;
if (!ostree_repo_resolve_rev (pull_data->repo, ref_with_collection->ref_name, TRUE,
&from_rev, error))
goto out;
/* Explicitly store NULL if there's no previous revision. We do
* this so we can assert() if we somehow didn't find a ref in the
* hash at all. Note we don't copy the collection-ref, so the
* lifetime of this hash must be equal to `requested_refs_to_fetch`.
*/
g_hash_table_insert (pull_data->ref_original_commits, ref_with_collection,
g_steal_pointer (&from_rev));
}
g_hash_table_replace (updated_requested_refs_to_fetch,
g_steal_pointer (&ref_with_collection),
g_steal_pointer (&checksum));
}
/* Resolve refs to a checksum if necessary */
if (pull_data->timestamp_check_from_rev &&
!ostree_validate_checksum_string (pull_data->timestamp_check_from_rev, NULL))
{
g_autofree char *from_rev = NULL;
if (!ostree_repo_resolve_rev (pull_data->repo, pull_data->timestamp_check_from_rev, FALSE,
&from_rev, error))
goto out;
g_free (pull_data->timestamp_check_from_rev);
pull_data->timestamp_check_from_rev = g_steal_pointer (&from_rev);
}
g_hash_table_unref (requested_refs_to_fetch);
requested_refs_to_fetch = g_steal_pointer (&updated_requested_refs_to_fetch);
if (g_hash_table_size (requested_refs_to_fetch) == 1)
{
GLNX_HASH_TABLE_FOREACH (requested_refs_to_fetch,
const OstreeCollectionRef *, ref)
{
the_ref_to_fetch = ref->ref_name;
break;
}
}
if (opt_ref_keyring_map_set)
{
const gchar *collection_id, *ref_name, *keyring_remote_name;
while (g_variant_iter_loop (ref_keyring_map_iter, "(&s&s&s)", &collection_id, &ref_name, &keyring_remote_name))
{
g_autoptr(OstreeCollectionRef) c_r = NULL;
if (!ostree_validate_collection_id (collection_id, error))
goto out;
if (!ostree_validate_rev (ref_name, error))
goto out;
if (!ostree_validate_remote_name (keyring_remote_name, error))
goto out;
c_r = ostree_collection_ref_new (collection_id, ref_name);
if (!g_hash_table_contains (requested_refs_to_fetch, c_r))
continue;
g_hash_table_insert (pull_data->ref_keyring_map,
g_steal_pointer (&c_r),
g_strdup (keyring_remote_name));
}
}
/* Create the state directory here - it's new with the commitpartial code,
* and may not exist in older repositories.
*/
if (mkdirat (pull_data->repo->repo_dir_fd, "state", 0777) != 0)
{
if (G_UNLIKELY (errno != EEXIST))
{
glnx_set_error_from_errno (error);
goto out;
}
}
pull_data->phase = OSTREE_PULL_PHASE_FETCHING_OBJECTS;
/* Now discard the previous fetcher, as it was bound to a temporary main context
* for synchronous requests.
*/
if (!reinitialize_fetcher (pull_data, remote_name_or_baseurl, error))
goto out;
pull_data->legacy_transaction_resuming = FALSE;
if (!inherit_transaction &&
!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming,
cancellable, error))
goto out;
if (pull_data->legacy_transaction_resuming)
g_debug ("resuming legacy transaction");
/* Initiate requests for explicit commit revisions */
GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit)
{
if (!initiate_request (pull_data, NULL, commit, error))
goto out;
}
/* Initiate requests for refs */
GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref,
const char*, to_revision)
{
if (!initiate_request (pull_data, ref, to_revision, error))
goto out;
}
if (pull_data->progress)
{
/* Setup a custom frequency if set */
if (update_frequency > 0)
update_timeout = g_timeout_source_new (pull_data->dry_run ? 0 : update_frequency);
else
update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
g_source_attach (update_timeout, pull_data->main_context);
}
/* Now await work completion */
while (!pull_termination_condition (pull_data))
g_main_context_iteration (pull_data->main_context, TRUE);
if (pull_data->caught_error)
goto out;
if (pull_data->dry_run)
{
ret = TRUE;
goto out;
}
g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0);
g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0);
g_assert_cmpint (pull_data->n_outstanding_content_fetches, ==, 0);
g_assert_cmpint (pull_data->n_outstanding_content_write_requests, ==, 0);
GLNX_HASH_TABLE_FOREACH_KV (requested_refs_to_fetch, const OstreeCollectionRef*, ref,
const char*, checksum)
{
g_autofree char *remote_ref = NULL;
g_autofree char *original_rev = NULL;
if (pull_data->remote_name)
remote_ref = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name);
else
remote_ref = g_strdup (ref->ref_name);
if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error))
goto out;
if (original_rev && strcmp (checksum, original_rev) == 0)
{
}
else
{
if (pull_data->is_mirror)
ostree_repo_transaction_set_collection_ref (pull_data->repo,
ref, checksum);
else
ostree_repo_transaction_set_ref (pull_data->repo,
pull_data->remote_refspec_name ?: pull_data->remote_name,
ref->ref_name, checksum);
}
}
if (pull_data->is_mirror && pull_data->summary_data &&
!refs_to_fetch && !opt_collection_refs_set && !configured_branches)
{
GLnxFileReplaceFlags replaceflag =
pull_data->repo->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : 0;
gsize len;
const guint8 *buf = g_bytes_get_data (pull_data->summary_data, &len);
if (!glnx_file_replace_contents_at (pull_data->repo->repo_dir_fd, "summary",
buf, len, replaceflag,
cancellable, error))
goto out;
if (pull_data->summary_data_sig)
{
buf = g_bytes_get_data (pull_data->summary_data_sig, &len);
if (!glnx_file_replace_contents_at (pull_data->repo->repo_dir_fd, "summary.sig",
buf, len, replaceflag,
cancellable, error))
goto out;
}
}
if (!inherit_transaction &&
!ostree_repo_commit_transaction (pull_data->repo, &tstats, cancellable, error))
goto out;
end_time = g_get_monotonic_time ();
bytes_transferred = _ostree_fetcher_bytes_transferred (pull_data->fetcher);
if (pull_data->progress)
{
g_autoptr(GString) buf = g_string_new ("");
/* Ensure the rest of the progress keys are set appropriately. */
update_progress (pull_data);
/* See if we did a local-only import */
if (pull_data->remote_repo_local)
g_string_append_printf (buf, "%u metadata, %u content objects imported",
pull_data->n_imported_metadata, pull_data->n_imported_content);
else if (pull_data->n_fetched_deltaparts > 0)
g_string_append_printf (buf, "%u delta parts, %u loose fetched",
pull_data->n_fetched_deltaparts,
pull_data->n_fetched_metadata + pull_data->n_fetched_content);
else
g_string_append_printf (buf, "%u metadata, %u content objects fetched",
pull_data->n_fetched_metadata, pull_data->n_fetched_content);
if (!pull_data->remote_repo_local &&
(pull_data->n_imported_metadata || pull_data->n_imported_content))
g_string_append_printf (buf, " (%u meta, %u content local)",
pull_data->n_imported_metadata,
pull_data->n_imported_content);
if (bytes_transferred > 0)
{
guint shift;
if (bytes_transferred < 1024)
shift = 1;
else
shift = 1024;
g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds",
(guint64)(bytes_transferred / shift),
shift == 1 ? "B" : "KiB",
(guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC));
}
if (!inherit_transaction)
{
g_autofree char *bytes_written = g_format_size (tstats.content_bytes_written);
g_string_append_printf (buf, "; %s content written", bytes_written);
}
ostree_async_progress_set_status (pull_data->progress, buf->str);
}
#ifdef HAVE_LIBSYSTEMD
if (bytes_transferred > 0 && pull_data->remote_name)
{
g_autoptr(GString) msg = g_string_new ("");
if (the_ref_to_fetch)
g_string_append_printf (msg, "libostree pull from '%s' for %s complete",
pull_data->remote_name, the_ref_to_fetch);
else
g_string_append_printf (msg, "libostree pull from '%s' for %u refs complete",
pull_data->remote_name, g_hash_table_size (requested_refs_to_fetch));
const char *gpg_verify_state;
#ifndef OSTREE_DISABLE_GPGME
if (pull_data->gpg_verify_summary)
{
if (pull_data->gpg_verify)
gpg_verify_state = "summary+commit";
else
gpg_verify_state = "summary-only";
}
else
gpg_verify_state = (pull_data->gpg_verify ? "commit" : "disabled");
#else
gpg_verify_state = "disabled";
#endif /* OSTREE_DISABLE_GPGME */
g_string_append_printf (msg, "\nsecurity: GPG: %s ", gpg_verify_state);
const char *sign_verify_state;
sign_verify_state = (pull_data->signapi_commit_verifiers ? "commit" : "disabled");
g_string_append_printf (msg, "\nsecurity: SIGN: %s ", sign_verify_state);
OstreeFetcherURI *first_uri = pull_data->meta_mirrorlist->pdata[0];
g_autofree char *first_scheme = _ostree_fetcher_uri_get_scheme (first_uri);
if (g_str_has_prefix (first_scheme, "http"))
{
g_string_append (msg, "http: ");
switch (pull_data->fetcher_security_state)
{
case OSTREE_FETCHER_SECURITY_STATE_CA_PINNED:
g_string_append (msg, "CA-pinned");
break;
case OSTREE_FETCHER_SECURITY_STATE_TLS:
g_string_append (msg, "TLS");
break;
case OSTREE_FETCHER_SECURITY_STATE_INSECURE:
g_string_append (msg, "insecure");
break;
}
}
g_string_append (msg, "\n");
if (pull_data->n_fetched_deltaparts > 0)
g_string_append_printf (msg, "delta: parts: %u loose: %u",
pull_data->n_fetched_deltaparts,
pull_data->n_fetched_metadata + pull_data->n_fetched_content);
else
g_string_append_printf (msg, "non-delta: meta: %u content: %u",
pull_data->n_fetched_metadata, pull_data->n_fetched_content);
const guint n_seconds = (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC);
g_autofree char *formatted_xferred = g_format_size (bytes_transferred);
g_string_append_printf (msg, "\ntransfer: secs: %u size: %s", n_seconds, formatted_xferred);
if (pull_data->signapi_commit_verifiers)
{
g_assert_cmpuint (g_hash_table_size (pull_data->signapi_verified_commits), >, 0);
}
ot_journal_send ("MESSAGE=%s", msg->str,
"MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_MESSAGE_FETCH_COMPLETE_ID),
"OSTREE_REMOTE=%s", pull_data->remote_name,
"OSTREE_SIGN=%s", sign_verify_state,
"OSTREE_GPG=%s", gpg_verify_state,
"OSTREE_SECONDS=%u", n_seconds,
"OSTREE_XFER_SIZE=%s", formatted_xferred,
NULL);
}
#endif
/* iterate over commits fetched and delete any commitpartial files */
if (pull_data->dirs == NULL && !pull_data->is_commit_only)
{
GLNX_HASH_TABLE_FOREACH_V (requested_refs_to_fetch, const char*, checksum)
{
if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, FALSE, error))
goto out;
}
GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit)
{
if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error))
goto out;
}
/* and finally any parent commits we might also have pulled because of depth>0 */
GLNX_HASH_TABLE_FOREACH (pull_data->commit_to_depth, const char*, commit)
{
if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error))
goto out;
}
}
ret = TRUE;
out:
/* This is pretty ugly - we have two error locations, because we
* have a mix of synchronous and async code. Mixing them gets messy
* as we need to avoid overwriting errors.
*/
if (pull_data->cached_async_error && error && !*error)
g_propagate_error (error, pull_data->cached_async_error);
else
g_clear_error (&pull_data->cached_async_error);
if (!inherit_transaction)
ostree_repo_abort_transaction (pull_data->repo, cancellable, NULL);
g_main_context_unref (pull_data->main_context);
if (update_timeout)
g_source_destroy (update_timeout);
g_strfreev (configured_branches);
g_clear_object (&pull_data->fetcher);
g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref);
g_clear_object (&pull_data->cancellable);
g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref);
g_clear_object (&pull_data->remote_repo_local);
g_free (pull_data->remote_refspec_name);
g_free (pull_data->remote_name);
g_free (pull_data->append_user_agent);
g_clear_pointer (&pull_data->signapi_commit_verifiers, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->signapi_summary_verifiers, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
g_clear_pointer (&pull_data->static_delta_superblocks, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->commit_to_depth, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->fetched_detached_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->ref_original_commits, (GDestroyNotify) g_hash_table_unref);
g_free (pull_data->timestamp_check_from_rev);
g_clear_pointer (&pull_data->verified_commits, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->signapi_verified_commits, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->ref_keyring_map, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref);
g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);
g_queue_clear (&pull_data->scan_object_queue);
g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
g_clear_pointer (&pull_data->dirs, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
return ret;
}
/* Structure used in ostree_repo_find_remotes_async() which stores metadata
* about a given OSTree commit. This includes the metadata from the commit
* #GVariant, plus some working state which is used to work out which remotes
* have refs pointing to this commit. */
typedef struct
{
gchar *checksum; /* always set */
guint64 commit_size; /* always set */
guint64 timestamp; /* 0 for unknown */
GVariant *additional_metadata;
GArray *refs; /* (element-type gsize), indexes to refs which point to this commit on at least one remote */
} CommitMetadata;
static void
commit_metadata_free (CommitMetadata *info)
{
g_clear_pointer (&info->refs, g_array_unref);
g_free (info->checksum);
g_clear_pointer (&info->additional_metadata, g_variant_unref);
g_free (info);
}
G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitMetadata, commit_metadata_free)
static CommitMetadata *
commit_metadata_new (const gchar *checksum,
guint64 commit_size,
guint64 timestamp,
GVariant *additional_metadata)
{
g_autoptr(CommitMetadata) info = NULL;
info = g_new0 (CommitMetadata, 1);
info->checksum = g_strdup (checksum);
info->commit_size = commit_size;
info->timestamp = timestamp;
info->additional_metadata = (additional_metadata != NULL) ? g_variant_ref (additional_metadata) : NULL;
info->refs = g_array_new (FALSE, FALSE, sizeof (gsize));
return g_steal_pointer (&info);
}
/* Structure used in ostree_repo_find_remotes_async() to store a grid (or table)
* of pointers, indexed by rows and columns. Basically an encapsulated 2D array.
* See the comments in ostree_repo_find_remotes_async() for its semantics
* there. */
typedef struct
{
gsize width; /* pointers */
gsize height; /* pointers */
gconstpointer pointers[]; /* n_pointers = width * height */
} PointerTable;
static void
pointer_table_free (PointerTable *table)
{
g_free (table);
}
G_DEFINE_AUTOPTR_CLEANUP_FUNC (PointerTable, pointer_table_free)
/* Both dimensions are in numbers of pointers. */
static PointerTable *
pointer_table_new (gsize width,
gsize height)
{
g_autoptr(PointerTable) table = NULL;
g_return_val_if_fail (width > 0, NULL);
g_return_val_if_fail (height > 0, NULL);
g_return_val_if_fail (width <= (G_MAXSIZE - sizeof (PointerTable)) / sizeof (gconstpointer) / height, NULL);
table = g_malloc0 (sizeof (PointerTable) + sizeof (gconstpointer) * width * height);
table->width = width;
table->height = height;
return g_steal_pointer (&table);
}
static gconstpointer
pointer_table_get (const PointerTable *table,
gsize x,
gsize y)
{
g_return_val_if_fail (table != NULL, FALSE);
g_return_val_if_fail (x < table->width, FALSE);
g_return_val_if_fail (y < table->height, FALSE);
return table->pointers[table->width * y + x];
}
static void
pointer_table_set (PointerTable *table,
gsize x,
gsize y,
gconstpointer value)
{
g_return_if_fail (table != NULL);
g_return_if_fail (x < table->width);
g_return_if_fail (y < table->height);
table->pointers[table->width * y + x] = value;
}
/* Validate the given struct contains a valid collection ID and ref name. */
static gboolean
is_valid_collection_ref (const OstreeCollectionRef *ref)
{
return (ref != NULL &&
ostree_validate_rev (ref->ref_name, NULL) &&
ostree_validate_collection_id (ref->collection_id, NULL));
}
/* Validate @refs is non-%NULL, non-empty, and contains only valid collection
* and ref names. */
static gboolean
is_valid_collection_ref_array (const OstreeCollectionRef * const *refs)
{
gsize i;
if (refs == NULL || *refs == NULL)
return FALSE;
for (i = 0; refs[i] != NULL; i++)
{
if (!is_valid_collection_ref (refs[i]))
return FALSE;
}
return TRUE;
}
/* Validate @finders is non-%NULL, non-empty, and contains only valid
* #OstreeRepoFinder instances. */
static gboolean
is_valid_finder_array (OstreeRepoFinder **finders)
{
gsize i;
if (finders == NULL || *finders == NULL)
return FALSE;
for (i = 0; finders[i] != NULL; i++)
{
if (!OSTREE_IS_REPO_FINDER (finders[i]))
return FALSE;
}
return TRUE;
}
/* Closure used to carry inputs from ostree_repo_find_remotes_async() to
* find_remotes_cb(). */
typedef struct
{
OstreeCollectionRef **refs;
GVariant *options;
OstreeAsyncProgress *progress;
OstreeRepoFinder *default_finder_avahi;
guint n_network_retries;
} FindRemotesData;
static void
find_remotes_data_free (FindRemotesData *data)
{
g_clear_object (&data->default_finder_avahi);
g_clear_object (&data->progress);
g_clear_pointer (&data->options, g_variant_unref);
ostree_collection_ref_freev (data->refs);
g_free (data);
}
G_DEFINE_AUTOPTR_CLEANUP_FUNC (FindRemotesData, find_remotes_data_free)
static FindRemotesData *
find_remotes_data_new (const OstreeCollectionRef * const *refs,
GVariant *options,
OstreeAsyncProgress *progress,
OstreeRepoFinder *default_finder_avahi,
guint n_network_retries)
{
g_autoptr(FindRemotesData) data = NULL;
data = g_new0 (FindRemotesData, 1);
data->refs = ostree_collection_ref_dupv (refs);
data->options = (options != NULL) ? g_variant_ref (options) : NULL;
data->progress = (progress != NULL) ? g_object_ref (progress) : NULL;
data->default_finder_avahi = (default_finder_avahi != NULL) ? g_object_ref (default_finder_avahi) : NULL;
data->n_network_retries = n_network_retries;
return g_steal_pointer (&data);
}
static gchar *
uint64_secs_to_iso8601 (guint64 secs)
{
g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs);
if (dt != NULL)
return g_date_time_format (dt, "%FT%TZ");
else
return g_strdup ("invalid");
}
static gint
sort_results_cb (gconstpointer a,
gconstpointer b)
{
const OstreeRepoFinderResult **result_a = (const OstreeRepoFinderResult **) a;
const OstreeRepoFinderResult **result_b = (const OstreeRepoFinderResult **) b;
return ostree_repo_finder_result_compare (*result_a, *result_b);
}
static void
repo_finder_result_free0 (OstreeRepoFinderResult *result)
{
if (result == NULL)
return;
ostree_repo_finder_result_free (result);
}
static void find_remotes_cb (GObject *obj,
GAsyncResult *result,
gpointer user_data);
/**
* ostree_repo_find_remotes_async:
* @self: an #OstreeRepo
* @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for
* @options: (nullable): a GVariant `a{sv}` with an extensible set of flags
* @finders: (array zero-terminated=1) (transfer none): non-empty array of
* #OstreeRepoFinder instances to use, or %NULL to use the system defaults
* @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s
* progress, or %NULL
* @cancellable: (nullable): a #GCancellable, or %NULL
* @callback: asynchronous completion callback
* @user_data: data to pass to @callback
*
* Find reachable remote URIs which claim to provide any of the given named
* @refs. This will search for configured remotes (#OstreeRepoFinderConfig),
* mounted volumes (#OstreeRepoFinderMount) and (if enabled at compile time)
* local network peers (#OstreeRepoFinderAvahi). In order to use a custom
* configuration of #OstreeRepoFinder instances, call
* ostree_repo_finder_resolve_all_async() on them individually.
*
* Any remote which is found and which claims to support any of the given @refs
* will be returned in the results. It is possible that a remote claims to
* support a given ref, but turns out not to — it is not possible to verify this
* until ostree_repo_pull_from_remotes_async() is called.
*
* The returned results will be sorted with the most useful first — this is
* typically the remote which claims to provide the most of @refs, at the lowest
* latency.
*
* Each result contains a list of the subset of @refs it claims to provide. It
* is possible for a non-empty list of results to be returned, but for some of
* @refs to not be listed in any of the results. Callers must check for this.
*
* Pass the results to ostree_repo_pull_from_remotes_async() to pull the given @refs
* from those remotes.
*
* The following @options are currently defined:
*
* * `override-commit-ids` (`as`): Array of specific commit IDs to fetch. The nth
* commit ID applies to the nth ref, so this must be the same length as @refs, if
* provided.
* * `n-network-retries` (`u`): Number of times to retry each download on
* receiving a transient network error, such as a socket timeout; default is
* 5, 0 means return errors without retrying. Since: 2018.6
*
* @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder
* instances to use, or %NULL to use the system default set of finders, which
* will typically be all available finders using their default options (but
* this is not guaranteed).
*
* GPG verification of commits will be used unconditionally.
*
* This will use the thread-default #GMainContext, but will not iterate it.
*
* Since: 2018.6
*/
void
ostree_repo_find_remotes_async (OstreeRepo *self,
const OstreeCollectionRef * const *refs,
GVariant *options,
OstreeRepoFinder **finders,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
g_autoptr(GTask) task = NULL;
g_autoptr(FindRemotesData) data = NULL;
OstreeRepoFinder *default_finders[4] = { NULL, };
g_autoptr(OstreeRepoFinder) finder_config = NULL;
g_autoptr(OstreeRepoFinder) finder_mount = NULL;
g_autoptr(OstreeRepoFinder) finder_avahi = NULL;
g_autofree char **override_commit_ids = NULL;
guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
g_return_if_fail (OSTREE_IS_REPO (self));
g_return_if_fail (is_valid_collection_ref_array (refs));
g_return_if_fail (options == NULL ||
g_variant_is_of_type (options, G_VARIANT_TYPE_VARDICT));
g_return_if_fail (finders == NULL || is_valid_finder_array (finders));
g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress));
g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable));
if (options)
{
(void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
g_return_if_fail (override_commit_ids == NULL || g_strv_length ((gchar **) refs) == g_strv_length (override_commit_ids));
(void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries);
}
/* Set up a task for the whole operation. */
task = g_task_new (self, cancellable, callback, user_data);
g_task_set_source_tag (task, ostree_repo_find_remotes_async);
/* Are we using #OstreeRepoFinders provided by the user, or the defaults? */
if (finders == NULL)
{
guint finder_index = 0;
#ifdef HAVE_AVAHI
guint avahi_index;
GMainContext *context = g_main_context_get_thread_default ();
g_autoptr(GError) local_error = NULL;
#endif /* HAVE_AVAHI */
if (g_strv_contains ((const char * const *)self->repo_finders, "config"))
default_finders[finder_index++] = finder_config = OSTREE_REPO_FINDER (ostree_repo_finder_config_new ());
if (g_strv_contains ((const char * const *)self->repo_finders, "mount"))
default_finders[finder_index++] = finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL));
#ifdef HAVE_AVAHI
if (g_strv_contains ((const char * const *)self->repo_finders, "lan"))
{
avahi_index = finder_index;
default_finders[finder_index++] = finder_avahi = OSTREE_REPO_FINDER (ostree_repo_finder_avahi_new (context));
}
#endif /* HAVE_AVAHI */
/* self->repo_finders is guaranteed to be non-empty */
g_assert (default_finders != NULL);
finders = default_finders;
#ifdef HAVE_AVAHI
if (finder_avahi != NULL)
{
ostree_repo_finder_avahi_start (OSTREE_REPO_FINDER_AVAHI (finder_avahi),
&local_error);
if (local_error != NULL)
{
/* See ostree-repo-finder-avahi.c:ostree_repo_finder_avahi_start, we
* intentionally throw this so as to distinguish between the Avahi
* finder failing because the Avahi daemon wasn't running and
* the Avahi finder failing because of some actual error.
*
* We need to distinguish between g_debug and g_warning here because
* unit tests that use this code may set G_DEBUG=fatal-warnings which
* would cause client code to abort if a warning were emitted.
*/
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
g_debug ("Avahi finder failed under normal operation; removing it: %s", local_error->message);
else
g_warning ("Avahi finder failed abnormally; removing it: %s", local_error->message);
default_finders[avahi_index] = NULL;
g_clear_object (&finder_avahi);
}
}
#endif /* HAVE_AVAHI */
}
/* We need to keep a pointer to the default Avahi finder so we can stop it
* again after the operation, which happens implicitly by dropping the final
* ref. */
data = find_remotes_data_new (refs, options, progress, finder_avahi, n_network_retries);
g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free);
/* Asynchronously resolve all possible remotes for the given refs. */
ostree_repo_finder_resolve_all_async (finders, refs, self, cancellable,
find_remotes_cb, g_steal_pointer (&task));
}
/* Find the first instance of (@collection_id, @ref_name) in @refs and return
* its index; or return %FALSE if nothing’s found. */
static gboolean
collection_refv_contains (const OstreeCollectionRef * const *refs,
const gchar *collection_id,
const gchar *ref_name,
gsize *out_index)
{
gsize i;
for (i = 0; refs[i] != NULL; i++)
{
if (g_str_equal (refs[i]->collection_id, collection_id) &&
g_str_equal (refs[i]->ref_name, ref_name))
{
*out_index = i;
return TRUE;
}
}
return FALSE;
}
/* For each ref from @refs which is listed in @summary_refs, cache its metadata
* from the summary file entry into @commit_metadatas, and add the checksum it
* points to into @refs_and_remotes_table at (@ref_index, @result_index).
* @ref_index is the ref’s index in @refs. */
static gboolean
find_remotes_process_refs (OstreeRepo *self,
const OstreeCollectionRef * const *refs,
OstreeRepoFinderResult *result,
gsize result_index,
const gchar *summary_collection_id,
GVariant *summary_refs,
GHashTable *commit_metadatas,
PointerTable *refs_and_remotes_table)
{
gsize j, n;
for (j = 0, n = g_variant_n_children (summary_refs); j < n; j++)
{
const guchar *csum_bytes;
g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL, stored_commit_v = NULL;
guint64 commit_size, commit_timestamp;
gchar tmp_checksum[OSTREE_SHA256_STRING_LEN + 1];
gsize ref_index;
g_autoptr(GDateTime) dt = NULL;
g_autoptr(GError) error = NULL;
const gchar *ref_name;
CommitMetadata *commit_metadata;
/* Check the ref name. */
ref_v = g_variant_get_child_value (summary_refs, j);
g_variant_get_child (ref_v, 0, "&s", &ref_name);
if (!ostree_validate_rev (ref_name, &error))
{
g_debug ("%s: Summary for result ‘%s’ contained invalid ref name ‘%s’: %s",
G_STRFUNC, result->remote->name, ref_name, error->message);
return FALSE;
}
/* Check the commit checksum. */
g_variant_get_child (ref_v, 1, "(t@ay@a{sv})", &commit_size, &csum_v, &commit_metadata_v);
csum_bytes = ostree_checksum_bytes_peek_validate (csum_v, &error);
if (csum_bytes == NULL)
{
g_debug ("%s: Summary for result ‘%s’ contained invalid ref checksum: %s",
G_STRFUNC, result->remote->name, error->message);
return FALSE;
}
ostree_checksum_inplace_from_bytes (csum_bytes, tmp_checksum);
/* Is this a ref we care about? */
if (!collection_refv_contains (refs, summary_collection_id, ref_name, &ref_index))
continue;
/* Load the commit from disk if possible, for verification. */
if (!ostree_repo_load_commit (self, tmp_checksum, &stored_commit_v, NULL, NULL))
stored_commit_v = NULL;
/* Check the additional metadata. */
if (!g_variant_lookup (commit_metadata_v, OSTREE_COMMIT_TIMESTAMP, "t", &commit_timestamp))
commit_timestamp = 0; /* unknown */
else
commit_timestamp = GUINT64_FROM_BE (commit_timestamp);
dt = g_date_time_new_from_unix_utc (commit_timestamp);
if (dt == NULL)
{
g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.",
G_STRFUNC, result->remote->name, commit_timestamp);
commit_timestamp = 0;
}
/* Check and store the commit metadata. */
commit_metadata = g_hash_table_lookup (commit_metadatas, tmp_checksum);
if (commit_metadata == NULL)
{
commit_metadata = commit_metadata_new (tmp_checksum, commit_size,
(stored_commit_v != NULL) ? ostree_commit_get_timestamp (stored_commit_v) : 0,
NULL);
g_hash_table_insert (commit_metadatas, commit_metadata->checksum,
commit_metadata /* transfer */);
}
/* Update the metadata if possible. */
if (commit_metadata->timestamp == 0)
{
commit_metadata->timestamp = commit_timestamp;
}
else if (commit_timestamp != 0 && commit_metadata->timestamp != commit_timestamp)
{
g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which did not match existing timestamp %" G_GUINT64_FORMAT ". Ignoring.",
G_STRFUNC, result->remote->name, commit_timestamp, commit_metadata->timestamp);
return FALSE;
}
if (commit_size != commit_metadata->commit_size)
{
g_debug ("%s: Summary for result ‘%s’ contained commit size %" G_GUINT64_FORMAT "B which did not match existing size %" G_GUINT64_FORMAT "B. Ignoring.",
G_STRFUNC, result->remote->name, commit_size, commit_metadata->commit_size);
return FALSE;
}
pointer_table_set (refs_and_remotes_table, ref_index, result_index, commit_metadata->checksum);
g_array_append_val (commit_metadata->refs, ref_index);
g_debug ("%s: Remote ‘%s’ lists ref ‘%s’ mapping to commit ‘%s’.",
G_STRFUNC, result->remote->name, ref_name, commit_metadata->checksum);
}
return TRUE;
}
static void
find_remotes_cb (GObject *obj,
GAsyncResult *async_result,
gpointer user_data)
{
OstreeRepo *self;
g_autoptr(GTask) task = NULL;
GCancellable *cancellable;
const FindRemotesData *data;
const OstreeCollectionRef * const *refs;
/* FIXME: We currently do nothing with @progress. Comment out to assuage -Wunused-variable */
/* OstreeAsyncProgress *progress; */
g_autoptr(GError) error = NULL;
g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */
gsize i;
g_autoptr(PointerTable) refs_and_remotes_table = NULL; /* (element-type commit-checksum) */
g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */
g_autoptr(OstreeFetcher) fetcher = NULL;
g_autofree const gchar **ref_to_latest_commit = NULL; /* indexed as @refs; (element-type commit-checksum) */
g_autofree guint64 *ref_to_latest_timestamp = NULL; /* indexed as @refs; (element-type commit-timestamp) */
gsize n_refs;
g_autofree char **override_commit_ids = NULL;
g_autoptr(GPtrArray) remotes_to_remove = NULL; /* (element-type OstreeRemote) */
g_autoptr(GPtrArray) final_results = NULL; /* (element-type OstreeRepoFinderResult) */
task = G_TASK (user_data);
self = OSTREE_REPO (g_task_get_source_object (task));
cancellable = g_task_get_cancellable (task);
data = g_task_get_task_data (task);
refs = (const OstreeCollectionRef * const *) data->refs;
/* progress = data->progress; */
/* Finish finding the remotes. */
results = ostree_repo_finder_resolve_all_finish (async_result, &error);
if (results == NULL)
{
g_task_return_error (task, g_steal_pointer (&error));
return;
}
if (results->len == 0)
{
g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref);
return;
}
/* Throughout this function, we eliminate invalid results from @results by
* clearing them to %NULL. We cannot remove them from the array, as that messes
* up iteration and stored array indices. Accordingly, we need the free function
* to be %NULL-safe. */
g_ptr_array_set_free_func (results, (GDestroyNotify) repo_finder_result_free0);
if (data->options)
{
(void) g_variant_lookup (data->options, "override-commit-ids", "^a&s", &override_commit_ids);
}
/* FIXME: In future, we also want to pull static delta superblocks in this
* phase, so that we have all the metadata we need for accurate size
* estimation for the actual pull operation. This should check the
* disable-static-deltas option first. */
/* Each key must be a pointer to the #CommitMetadata.checksum field of its value. */
commit_metadatas = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) commit_metadata_free);
/* X dimension is an index into @refs. Y dimension is an index into @results.
* Each cell stores the commit checksum which that ref resolves to on that
* remote, or %NULL if the remote doesn’t have that ref. */
n_refs = g_strv_length ((gchar **) refs); /* it’s not a GStrv, but this works */
refs_and_remotes_table = pointer_table_new (n_refs, results->len);
remotes_to_remove = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_remote_unref);
/* Fetch and validate the summary file for each result. */
/* FIXME: All these downloads could be parallelised; that requires the
* ostree_repo_remote_fetch_summary_with_options() API to be async. */
for (i = 0; i < results->len; i++)
{
OstreeRepoFinderResult *result = g_ptr_array_index (results, i);
g_autoptr(GBytes) summary_bytes = NULL;
g_autoptr(GVariant) summary_v = NULL;
guint64 summary_last_modified;
g_autoptr(GVariant) summary_refs = NULL;
g_autoptr(GVariant) additional_metadata_v = NULL;
g_autofree gchar *summary_collection_id = NULL;
g_autoptr(GVariantIter) summary_collection_map = NULL;
gboolean invalid_result = FALSE;
/* Add the remote to our internal list of remotes, so other libostree
* API can access it. */
if (!_ostree_repo_add_remote (self, result->remote))
g_ptr_array_add (remotes_to_remove, ostree_remote_ref (result->remote));
g_debug ("%s: Fetching summary for remote ‘%s’ with keyring ‘%s’.",
G_STRFUNC, result->remote->name, result->remote->keyring);
/* Download the summary. This will load from the cache if possible. */
ostree_repo_remote_fetch_summary_with_options (self,
result->remote->name,
NULL, /* no options */
&summary_bytes,
NULL,
cancellable,
&error);
if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
goto error;
else if (error != NULL)
{
g_debug ("%s: Failed to download summary for result ‘%s’. Ignoring. %s",
G_STRFUNC, result->remote->name, error->message);
g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free);
g_clear_error (&error);
continue;
}
else if (summary_bytes == NULL)
{
g_debug ("%s: Failed to download summary for result ‘%s’. Ignoring. %s",
G_STRFUNC, result->remote->name,
"No summary file exists on server");
g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free);
continue;
}
/* Check the metadata in the summary file, especially whether it contains
* all the @refs we are interested in. */
summary_v = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
summary_bytes, FALSE));
/* Check the summary’s additional metadata and set up @commit_metadata
* and @refs_and_remotes_table with the refs listed in the summary file,
* filtered by the keyring associated with this result and the
* intersection with @refs. */
additional_metadata_v = g_variant_get_child_value (summary_v, 1);
if (g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_ID, "s", &summary_collection_id))
{
summary_refs = g_variant_get_child_value (summary_v, 0);
if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs,
commit_metadatas, refs_and_remotes_table))
{
g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free);
continue;
}
}
if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_MAP, "a{sa(s(taya{sv}))}", &summary_collection_map))
summary_collection_map = NULL;
while (summary_collection_map != NULL &&
g_variant_iter_loop (summary_collection_map, "{s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs))
{
/* Exclude refs that don't use the associated keyring if this is a
* dynamic remote, by comparing against the collection ID of the
* remote this one inherits from */
if (result->remote->refspec_name != NULL &&
!check_remote_matches_collection_id (self, result->remote->refspec_name, summary_collection_id))
continue;
if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs,
commit_metadatas, refs_and_remotes_table))
{
g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free);
invalid_result = TRUE;
break;
}
}
if (invalid_result)
continue;
/* Check the summary timestamp. */
if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_LAST_MODIFIED, "t", &summary_last_modified))
summary_last_modified = 0;
else
summary_last_modified = GUINT64_FROM_BE (summary_last_modified);
/* Update the stored result data. Clear the @ref_to_checksum map, since
* it’s been moved to @refs_and_remotes_table and is now potentially out
* of date. */
g_clear_pointer (&result->ref_to_checksum, g_hash_table_unref);
g_clear_pointer (&result->ref_to_timestamp, g_hash_table_unref);
result->summary_last_modified = summary_last_modified;
}
/* Fill in any gaps in the metadata for the most recent commits by pulling
* the commit metadata from the remotes. The ‘most recent commits’ are the
* set of head commits pointed to by the refs we just resolved from the
* summary files. */
GLNX_HASH_TABLE_FOREACH_V (commit_metadatas, CommitMetadata*, commit_metadata)
{
char buf[_OSTREE_LOOSE_PATH_MAX];
g_autofree gchar *commit_filename = NULL;
g_autoptr(GPtrArray) mirrorlist = NULL; /* (element-type OstreeFetcherURI) */
g_autoptr(GBytes) commit_bytes = NULL;
g_autoptr(GVariant) commit_v = NULL;
guint64 commit_timestamp;
g_autoptr(GDateTime) dt = NULL;
/* Already complete? */
if (commit_metadata->timestamp != 0)
continue;
_ostree_loose_path (buf, commit_metadata->checksum, OSTREE_OBJECT_TYPE_COMMIT, OSTREE_REPO_MODE_ARCHIVE);
commit_filename = g_build_filename ("objects", buf, NULL);
/* For each of the remotes whose summary files contain this ref, try
* downloading the commit metadata until we succeed. Since the results are
* in priority order, the most important remotes are tried first. */
for (i = 0; i < commit_metadata->refs->len; i++)
{
gsize ref_index = g_array_index (commit_metadata->refs, gsize, i);
gsize j;
for (j = 0; j < results->len; j++)
{
OstreeRepoFinderResult *result = g_ptr_array_index (results, j);
/* Previous error processing this result? */
if (result == NULL)
continue;
if (pointer_table_get (refs_and_remotes_table, ref_index, j) != commit_metadata->checksum)
continue;
g_autofree gchar *uri = NULL;
g_autoptr(OstreeFetcherURI) fetcher_uri = NULL;
if (!ostree_repo_remote_get_url (self, result->remote->name,
&uri, &error))
goto error;
fetcher_uri = _ostree_fetcher_uri_parse (uri, &error);
if (fetcher_uri == NULL)
goto error;
fetcher = _ostree_repo_remote_new_fetcher (self, result->remote->name,
TRUE, NULL, NULL, NULL, &error);
if (fetcher == NULL)
goto error;
g_debug ("%s: Fetching metadata for commit ‘%s’ from remote ‘%s’.",
G_STRFUNC, commit_metadata->checksum, result->remote->name);
/* FIXME: Support remotes which have contenturl, mirrorlist, etc. */
mirrorlist = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (mirrorlist, g_steal_pointer (&fetcher_uri));
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher,
mirrorlist,
commit_filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
data->n_network_retries,
&commit_bytes,
0, /* no maximum size */
cancellable,
&error))
goto error;
g_autoptr(OstreeGpgVerifyResult) verify_result = NULL;
verify_result = ostree_repo_verify_commit_for_remote (self,
commit_metadata->checksum,
result->remote->name,
cancellable,
&error);
if (verify_result == NULL)
{
g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum);
goto error;
}
if (!ostree_gpg_verify_result_require_valid_signature (verify_result, &error))
{
g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum);
goto error;
}
if (commit_bytes != NULL)
break;
}
if (commit_bytes != NULL)
break;
}
if (commit_bytes == NULL)
{
g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Metadata not found for commit ‘%s’", commit_metadata->checksum);
goto error;
}
/* Parse the commit metadata. */
commit_v = g_variant_new_from_bytes (OSTREE_COMMIT_GVARIANT_FORMAT,
commit_bytes, FALSE);
g_variant_get_child (commit_v, 5, "t", &commit_timestamp);
commit_timestamp = GUINT64_FROM_BE (commit_timestamp);
dt = g_date_time_new_from_unix_utc (commit_timestamp);
if (dt == NULL)
{
g_debug ("%s: Commit ‘%s’ metadata contained timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.",
G_STRFUNC, commit_metadata->checksum, commit_timestamp);
commit_timestamp = 0;
}
/* Update the #CommitMetadata. */
commit_metadata->timestamp = commit_timestamp;
}
/* Find the latest commit for each ref. This is where we resolve the
* differences between remotes: two remotes could both contain ref R, but one
* remote could be outdated compared to the other, and point to an older
* commit. For each ref, we want to find the most recent commit any remote
* points to for it (unless override-commit-ids was used).
*
* @ref_to_latest_commit is indexed by @ref_index, and its values are the
* latest checksum for each ref. If override-commit-ids was used,
* @ref_to_latest_commit won't be initialized or used.
*
* @ref_to_latest_timestamp is also indexed by @ref_index, and its values are
* the latest timestamp for each ref, when available.*/
ref_to_latest_commit = g_new0 (const gchar *, n_refs);
ref_to_latest_timestamp = g_new0 (guint64, n_refs);
for (i = 0; i < n_refs; i++)
{
gsize j;
const gchar *latest_checksum = NULL;
const CommitMetadata *latest_commit_metadata = NULL;
g_autofree gchar *latest_commit_timestamp_str = NULL;
if (override_commit_ids)
{
g_debug ("%s: Using specified commit ‘%s’ for ref (%s, %s).",
G_STRFUNC, override_commit_ids[i], refs[i]->collection_id, refs[i]->ref_name);
continue;
}
for (j = 0; j < results->len; j++)
{
const CommitMetadata *candidate_commit_metadata;
const gchar *candidate_checksum;
candidate_checksum = pointer_table_get (refs_and_remotes_table, i, j);
if (candidate_checksum == NULL)
continue;
candidate_commit_metadata = g_hash_table_lookup (commit_metadatas, candidate_checksum);
g_assert (candidate_commit_metadata != NULL);
if (latest_commit_metadata == NULL ||
candidate_commit_metadata->timestamp > latest_commit_metadata->timestamp)
{
latest_checksum = candidate_checksum;
latest_commit_metadata = candidate_commit_metadata;
}
}
/* @latest_checksum could be %NULL here if there was an error downloading
* the summary or commit metadata files above. */
ref_to_latest_commit[i] = latest_checksum;
if (latest_checksum != NULL && latest_commit_metadata != NULL)
ref_to_latest_timestamp[i] = latest_commit_metadata->timestamp;
else
ref_to_latest_timestamp[i] = 0;
if (latest_commit_metadata != NULL)
{
latest_commit_timestamp_str = uint64_secs_to_iso8601 (latest_commit_metadata->timestamp);
g_debug ("%s: Latest commit for ref (%s, %s) across all remotes is ‘%s’ with timestamp %s.",
G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name,
latest_checksum, latest_commit_timestamp_str);
}
else
{
g_debug ("%s: Latest commit for ref (%s, %s) is unknown due to failure to download metadata.",
G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name);
}
}
/* Recombine @commit_metadatas and @results so that each
* #OstreeRepoFinderResult.refs lists the refs for which that remote has the
* latest commits (i.e. it’s not out of date compared to some other remote). */
final_results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free);
for (i = 0; i < results->len; i++)
{
OstreeRepoFinderResult *result = g_ptr_array_index (results, i);
g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type OstreeCollectionRef utf8) */
g_autoptr(GHashTable) validated_ref_to_timestamp = NULL; /* (element-type OstreeCollectionRef guint64) */
gsize j, n_latest_refs;
/* Previous error processing this result? */
if (result == NULL)
continue;
/* Map of refs to checksums provided by this result. The checksums should
* be %NULL for each ref unless this result provides the latest checksum. */
validated_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash,
ostree_collection_ref_equal,
(GDestroyNotify) ostree_collection_ref_free,
g_free);
validated_ref_to_timestamp = g_hash_table_new_full (ostree_collection_ref_hash,
ostree_collection_ref_equal,
(GDestroyNotify) ostree_collection_ref_free,
g_free);
if (override_commit_ids)
{
for (j = 0; refs[j] != NULL; j++)
{
guint64 *timestamp_ptr;
g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]),
g_strdup (override_commit_ids[j]));
timestamp_ptr = g_malloc (sizeof (guint64));
*timestamp_ptr = 0;
g_hash_table_insert (validated_ref_to_timestamp, ostree_collection_ref_dup (refs[j]),
timestamp_ptr);
}
}
else
{
n_latest_refs = 0;
for (j = 0; refs[j] != NULL; j++)
{
const gchar *latest_commit_for_ref = ref_to_latest_commit[j];
guint64 *timestamp_ptr;
if (pointer_table_get (refs_and_remotes_table, j, i) != latest_commit_for_ref)
latest_commit_for_ref = NULL;
if (latest_commit_for_ref != NULL)
n_latest_refs++;
g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]),
g_strdup (latest_commit_for_ref));
timestamp_ptr = g_malloc (sizeof (guint64));
if (latest_commit_for_ref != NULL)
*timestamp_ptr = GUINT64_TO_BE (ref_to_latest_timestamp[j]);
else
*timestamp_ptr = 0;
g_hash_table_insert (validated_ref_to_timestamp, ostree_collection_ref_dup (refs[j]),
timestamp_ptr);
}
if (n_latest_refs == 0)
{
g_debug ("%s: Omitting remote ‘%s’ from results as none of its refs are new enough.",
G_STRFUNC, result->remote->name);
ostree_repo_finder_result_free (g_steal_pointer (&g_ptr_array_index (results, i)));
continue;
}
}
result->ref_to_checksum = g_steal_pointer (&validated_ref_to_checksum);
result->ref_to_timestamp = g_steal_pointer (&validated_ref_to_timestamp);
g_ptr_array_add (final_results, g_steal_pointer (&g_ptr_array_index (results, i)));
}
/* Ensure the updated results are still in priority order. */
g_ptr_array_sort (final_results, sort_results_cb);
/* Remove the remotes we temporarily added.
* FIXME: It would be so much better if we could pass #OstreeRemote pointers
* around internally, to avoid serialising on the global table of them. */
for (i = 0; i < remotes_to_remove->len; i++)
{
OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i);
_ostree_repo_remove_remote (self, remote);
}
g_task_return_pointer (task, g_steal_pointer (&final_results), (GDestroyNotify) g_ptr_array_unref);
return;
error:
/* Remove the remotes we temporarily added. */
for (i = 0; i < remotes_to_remove->len; i++)
{
OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i);
_ostree_repo_remove_remote (self, remote);
}
g_task_return_error (task, g_steal_pointer (&error));
}
/**
* ostree_repo_find_remotes_finish:
* @self: an #OstreeRepo
* @result: the asynchronous result
* @error: return location for a #GError, or %NULL
*
* Finish an asynchronous pull operation started with
* ostree_repo_find_remotes_async().
*
* Returns: (transfer full) (array zero-terminated=1): a potentially empty array
* of #OstreeRepoFinderResults, followed by a %NULL terminator element; or
* %NULL on error
* Since: 2018.6
*/
OstreeRepoFinderResult **
ostree_repo_find_remotes_finish (OstreeRepo *self,
GAsyncResult *result,
GError **error)
{
g_autoptr(GPtrArray) results = NULL;
g_return_val_if_fail (OSTREE_IS_REPO (self), NULL);
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_find_remotes_async), NULL);
g_return_val_if_fail (error == NULL || *error == NULL, NULL);
results = g_task_propagate_pointer (G_TASK (result), error);
if (results != NULL)
{
g_ptr_array_add (results, NULL); /* NULL terminator */
return (OstreeRepoFinderResult **) g_ptr_array_free (g_steal_pointer (&results), FALSE);
}
else
return NULL;
}
static void
copy_option (GVariantDict *master_options,
GVariantDict *slave_options,
const gchar *key,
const GVariantType *expected_type)
{
g_autoptr(GVariant) option_v = g_variant_dict_lookup_value (master_options, key, expected_type);
if (option_v != NULL)
g_variant_dict_insert_value (slave_options, key, option_v);
}
/**
* ostree_repo_pull_from_remotes_async:
* @self: an #OstreeRepo
* @results: (array zero-terminated=1): %NULL-terminated array of remotes to
* pull from, including the refs to pull from each
* @options: (nullable): A GVariant `a{sv}` with an extensible set of flags
* @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s
* progress, or %NULL
* @cancellable: (nullable): a #GCancellable, or %NULL
* @callback: asynchronous completion callback
* @user_data: data to pass to @callback
*
* Pull refs from multiple remotes which have been found using
* ostree_repo_find_remotes_async().
*
* @results are expected to be in priority order, with the best remotes to pull
* from listed first. ostree_repo_pull_from_remotes_async() will generally pull
* from the remotes in order, but may parallelise its downloads.
*
* If an error is encountered when pulling from a given remote, that remote will
* be ignored and another will be tried instead. If any refs have not been
* downloaded successfully after all remotes have been tried, %G_IO_ERROR_FAILED
* will be returned. The results of any successful downloads will remain cached
* in the local repository.
*
* If @cancellable is cancelled, %G_IO_ERROR_CANCELLED will be returned
* immediately. The results of any successfully completed downloads at that
* point will remain cached in the local repository.
*
* GPG verification of commits will be used unconditionally.
*
* The following @options are currently defined:
*
* * `flags` (`i`): #OstreeRepoPullFlags to apply to the pull operation
* * `inherit-transaction` (`b`): %TRUE to inherit an ongoing transaction on
* the #OstreeRepo, rather than encapsulating the pull in a new one
* * `depth` (`i`): How far in the history to traverse; default is 0, -1 means infinite
* * `disable-static-deltas` (`b`): Do not use static deltas
* * `http-headers` (`a(ss)`): Additional headers to add to all HTTP requests
* * `subdirs` (`as`): Pull just these subdirectories
* * `update-frequency` (`u`): Frequency to call the async progress callback in
* milliseconds, if any; only values higher than 0 are valid
* * `append-user-agent` (`s`): Additional string to append to the user agent
* * `n-network-retries` (`u`): Number of times to retry each download on receiving
* a transient network error, such as a socket timeout; default is 5, 0
* means return errors without retrying. Since: 2018.6
* * `ref-keyring-map` (`a(sss)`): Array of (collection ID, ref name, keyring
* remote name) tuples specifying which remote's keyring should be used when
* doing GPG verification of each collection-ref. This is useful to prevent a
* remote from serving malicious updates to refs which did not originate from
* it. This can be a subset or superset of the refs being pulled; any ref
* not being pulled will be ignored and any ref without a keyring remote
* will be verified with the keyring of the remote being pulled from.
* Since: 2019.2
*
* Since: 2018.6
*/
void
ostree_repo_pull_from_remotes_async (OstreeRepo *self,
const OstreeRepoFinderResult * const *results,
GVariant *options,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
g_return_if_fail (OSTREE_IS_REPO (self));
g_return_if_fail (results != NULL && results[0] != NULL);
g_return_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}")));
g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress));
g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable));
g_autoptr(GTask) task = NULL;
g_autoptr(GHashTable) refs_pulled = NULL; /* (element-type OstreeCollectionRef gboolean) */
gsize i, j;
g_autoptr(GString) refs_unpulled_string = NULL;
g_autoptr(GError) local_error = NULL;
g_auto(GVariantDict) options_dict = OT_VARIANT_BUILDER_INITIALIZER;
OstreeRepoPullFlags flags;
gboolean inherit_transaction;
/* Set up a task for the whole operation. */
task = g_task_new (self, cancellable, callback, user_data);
g_task_set_source_tag (task, ostree_repo_pull_from_remotes_async);
/* Keep track of the set of refs we’ve pulled already. Value is %TRUE if the
* ref has been pulled; %FALSE if it has not. */
refs_pulled = g_hash_table_new_full (ostree_collection_ref_hash,
ostree_collection_ref_equal, NULL, NULL);
g_variant_dict_init (&options_dict, options);
if (!g_variant_dict_lookup (&options_dict, "flags", "i", &flags))
flags = OSTREE_REPO_PULL_FLAGS_NONE;
if (!g_variant_dict_lookup (&options_dict, "inherit-transaction", "b", &inherit_transaction))
inherit_transaction = FALSE;
/* Run all the local pull operations in a single overall transaction. */
if (!inherit_transaction &&
!ostree_repo_prepare_transaction (self, NULL, cancellable, &local_error))
{
g_task_return_error (task, g_steal_pointer (&local_error));
return;
}
/* FIXME: Rework this code to pull in parallel where possible. At the moment
* we expect the (i == 0) iteration will do all the work (all the refs) and
* subsequent iterations are only there in case of error.
*
* The code is currently all synchronous, too. Making it asynchronous requires
* the underlying pull code to be asynchronous. */
for (i = 0; results[i] != NULL; i++)
{
const OstreeRepoFinderResult *result = results[i];
g_autoptr(GString) refs_to_pull_str = NULL;
g_autoptr(GPtrArray) refs_to_pull = NULL; /* (element-type OstreeCollectionRef) */
g_auto(GVariantBuilder) refs_to_pull_builder = OT_VARIANT_BUILDER_INITIALIZER;
g_auto(GVariantDict) local_options_dict = OT_VARIANT_BUILDER_INITIALIZER;
g_autoptr(GVariant) local_options = NULL;
gboolean remove_remote;
refs_to_pull = g_ptr_array_new_with_free_func (NULL);
refs_to_pull_str = g_string_new ("");
g_variant_builder_init (&refs_to_pull_builder, G_VARIANT_TYPE ("a(sss)"));
GLNX_HASH_TABLE_FOREACH_KV (result->ref_to_checksum, const OstreeCollectionRef*, ref,
const char*, checksum)
{
if (checksum != NULL &&
!GPOINTER_TO_INT (g_hash_table_lookup (refs_pulled, ref)))
{
g_ptr_array_add (refs_to_pull, (gpointer) ref);
g_variant_builder_add (&refs_to_pull_builder, "(sss)",
ref->collection_id, ref->ref_name, checksum);
if (refs_to_pull_str->len > 0)
g_string_append (refs_to_pull_str, ", ");
g_string_append_printf (refs_to_pull_str, "(%s, %s)",
ref->collection_id, ref->ref_name);
}
}
if (refs_to_pull->len == 0)
{
g_debug ("Ignoring remote ‘%s’ as it has no relevant refs or they "
"have already been pulled.",
result->remote->name);
continue;
}
/* NULL terminators. */
g_ptr_array_add (refs_to_pull, NULL);
g_debug ("Pulling from remote ‘%s’: %s",
result->remote->name, refs_to_pull_str->str);
/* Set up the pull options. */
g_variant_dict_init (&local_options_dict, NULL);
g_variant_dict_insert (&local_options_dict, "flags", "i", OSTREE_REPO_PULL_FLAGS_UNTRUSTED | flags);
g_variant_dict_insert_value (&local_options_dict, "collection-refs", g_variant_builder_end (&refs_to_pull_builder));
#ifndef OSTREE_DISABLE_GPGME
g_variant_dict_insert (&local_options_dict, "gpg-verify", "b", TRUE);
#else
g_variant_dict_insert (&local_options_dict, "gpg-verify", "b", FALSE);
#endif /* OSTREE_DISABLE_GPGME */
g_variant_dict_insert (&local_options_dict, "gpg-verify-summary", "b", FALSE);
g_variant_dict_insert (&local_options_dict, "sign-verify", "b", FALSE);
g_variant_dict_insert (&local_options_dict, "sign-verify-summary", "b", FALSE);
g_variant_dict_insert (&local_options_dict, "inherit-transaction", "b", TRUE);
if (result->remote->refspec_name != NULL)
g_variant_dict_insert (&local_options_dict, "override-remote-name", "s", result->remote->refspec_name);
copy_option (&options_dict, &local_options_dict, "depth", G_VARIANT_TYPE ("i"));
copy_option (&options_dict, &local_options_dict, "disable-static-deltas", G_VARIANT_TYPE ("b"));
copy_option (&options_dict, &local_options_dict, "http-headers", G_VARIANT_TYPE ("a(ss)"));
copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as"));
copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u"));
copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s"));
copy_option (&options_dict, &local_options_dict, "n-network-retries", G_VARIANT_TYPE ("u"));
copy_option (&options_dict, &local_options_dict, "ref-keyring-map", G_VARIANT_TYPE ("a(sss)"));
local_options = g_variant_dict_end (&local_options_dict);
/* FIXME: We do nothing useful with @progress at the moment. */
remove_remote = !_ostree_repo_add_remote (self, result->remote);
ostree_repo_pull_with_options (self, result->remote->name, local_options,
progress, cancellable, &local_error);
if (remove_remote)
_ostree_repo_remove_remote (self, result->remote);
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED))
{
if (!inherit_transaction)
ostree_repo_abort_transaction (self, NULL, NULL);
g_task_return_error (task, g_steal_pointer (&local_error));
return;
}
for (j = 0; refs_to_pull->pdata[j] != NULL; j++)
g_hash_table_replace (refs_pulled, refs_to_pull->pdata[j],
GINT_TO_POINTER (local_error == NULL));
if (local_error != NULL)
{
g_debug ("Failed to pull refs from ‘%s’: %s",
result->remote->name, local_error->message);
g_clear_error (&local_error);
continue;
}
else
{
g_debug ("Pulled refs from ‘%s’.", result->remote->name);
}
}
/* Commit the transaction. */
if (!inherit_transaction &&
!ostree_repo_commit_transaction (self, NULL, cancellable, &local_error))
{
g_task_return_error (task, g_steal_pointer (&local_error));
return;
}
/* Any refs left un-downloaded? If so, we’ve failed. */
GLNX_HASH_TABLE_FOREACH_KV (refs_pulled, const OstreeCollectionRef*, ref,
gpointer, is_pulled_pointer)
{
gboolean is_pulled = GPOINTER_TO_INT (is_pulled_pointer);
if (is_pulled)
continue;
if (refs_unpulled_string == NULL)
refs_unpulled_string = g_string_new ("");
else
g_string_append (refs_unpulled_string, ", ");
g_string_append_printf (refs_unpulled_string, "(%s, %s)",
ref->collection_id, ref->ref_name);
}
if (refs_unpulled_string != NULL)
{
g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to pull some refs from the remotes: %s",
refs_unpulled_string->str);
return;
}
g_task_return_boolean (task, TRUE);
}
/**
* ostree_repo_pull_from_remotes_finish:
* @self: an #OstreeRepo
* @result: the asynchronous result
* @error: return location for a #GError, or %NULL
*
* Finish an asynchronous pull operation started with
* ostree_repo_pull_from_remotes_async().
*
* Returns: %TRUE on success, %FALSE otherwise
* Since: 2018.6
*/
gboolean
ostree_repo_pull_from_remotes_finish (OstreeRepo *self,
GAsyncResult *result,
GError **error)
{
g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_pull_from_remotes_async), FALSE);
g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
return g_task_propagate_boolean (G_TASK (result), error);
}
/**
* ostree_repo_remote_fetch_summary_with_options:
* @self: Self
* @name: name of a remote
* @options: (nullable): A GVariant a{sv} with an extensible set of flags
* @out_summary: (out) (optional): return location for raw summary data, or
* %NULL
* @out_signatures: (out) (optional): return location for raw summary
* signature data, or %NULL
* @cancellable: a #GCancellable
* @error: a #GError
*
* Like ostree_repo_remote_fetch_summary(), but supports an extensible set of flags.
* The following are currently defined:
*
* - override-url (s): Fetch summary from this URL if remote specifies no metalink in options
* - http-headers (a(ss)): Additional headers to add to all HTTP requests
* - append-user-agent (s): Additional string to append to the user agent
* - n-network-retries (u): Number of times to retry each download on receiving
* a transient network error, such as a socket timeout; default is 5, 0
* means return errors without retrying
*
* Returns: %TRUE on success, %FALSE on failure
*
* Since: 2016.6
*/
gboolean
ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
const char *name,
GVariant *options,
GBytes **out_summary,
GBytes **out_signatures,
GCancellable *cancellable,
GError **error)
{
g_autofree char *metalink_url_string = NULL;
g_autoptr(GBytes) summary = NULL;
g_autoptr(GBytes) signatures = NULL;
gboolean gpg_verify_summary;
g_autoptr(GPtrArray) signapi_summary_verifiers = NULL;
gboolean summary_is_from_cache = FALSE;
g_autoptr(OstreeFetcher) fetcher = NULL;
g_autoptr(GMainContextPopDefault) mainctx = NULL;
const char *url_override = NULL;
g_autoptr(GVariant) extra_headers = NULL;
g_autoptr(GPtrArray) mirrorlist = NULL;
const char *append_user_agent = NULL;
guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
g_return_val_if_fail (OSTREE_REPO (self), FALSE);
g_return_val_if_fail (name != NULL, FALSE);
if (!ostree_repo_get_remote_option (self, name, "metalink", NULL,
&metalink_url_string, error))
return FALSE;
if (options)
{
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
(void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers);
(void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent);
(void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries);
}
if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error))
return FALSE;
if (!_signapi_init_for_remote (self, name, NULL,
&signapi_summary_verifiers,
error))
return FALSE;
mainctx = _ostree_main_context_new_default ();
fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, extra_headers, append_user_agent, NULL, error);
if (fetcher == NULL)
return FALSE;
if (metalink_url_string)
{
g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (metalink_url_string, error);
if (!uri)
return FALSE;
mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free);
g_ptr_array_add (mirrorlist, g_steal_pointer (&uri));
}
else if (!compute_effective_mirrorlist (self, name, url_override,
fetcher, n_network_retries,
&mirrorlist, cancellable, error))
return FALSE;
/* FIXME: Send the ETag from the cache with the request for summary.sig to
* avoid downloading summary.sig unnecessarily. This won’t normally provide
* any benefits (but won’t do any harm) since summary.sig is typically 500B
* in size. But if a repository has multiple keys, the signature file will
* grow and this optimisation may be useful. */
if (!_ostree_preload_metadata_file (self,
fetcher,
mirrorlist,
"summary.sig",
metalink_url_string ? TRUE : FALSE,
n_network_retries,
&signatures,
cancellable,
error))
return FALSE;
if (signatures)
{
if (!_ostree_repo_load_cache_summary_if_same_sig (self,
name,
signatures,
&summary,
cancellable,
error))
return FALSE;
}
if (summary)
summary_is_from_cache = TRUE;
else
{
if (!_ostree_preload_metadata_file (self,
fetcher,
mirrorlist,
"summary",
metalink_url_string ? TRUE : FALSE,
n_network_retries,
&summary,
cancellable,
error))
return FALSE;
}
if (!_ostree_repo_verify_summary (self, name,
gpg_verify_summary, signapi_summary_verifiers,
summary, signatures,
cancellable, error))
return FALSE;
if (!summary_is_from_cache && summary && signatures)
{
g_autoptr(GError) temp_error = NULL;
if (!_ostree_repo_cache_summary (self,
name,
summary,
signatures,
cancellable,
&temp_error))
{
if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED))
g_debug ("No permissions to save summary cache");
else
{
g_propagate_error (error, g_steal_pointer (&temp_error));
return FALSE;
}
}
}
if (out_summary != NULL)
*out_summary = g_steal_pointer (&summary);
if (out_signatures != NULL)
*out_signatures = g_steal_pointer (&signatures);
return TRUE;
}
#else /* HAVE_LIBCURL_OR_LIBSOUP */
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
const char *remote_name_or_baseurl,
GVariant *options,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GError **error)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP");
return FALSE;
}
gboolean
ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
const char *name,
GVariant *options,
GBytes **out_summary,
GBytes **out_signatures,
GCancellable *cancellable,
GError **error)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP");
return FALSE;
}
void
ostree_repo_find_remotes_async (OstreeRepo *self,
const OstreeCollectionRef * const *refs,
GVariant *options,
OstreeRepoFinder **finders,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
g_return_if_fail (OSTREE_IS_REPO (self));
g_task_report_new_error (self, callback, user_data, ostree_repo_find_remotes_async,
G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP");
}
OstreeRepoFinderResult **
ostree_repo_find_remotes_finish (OstreeRepo *self,
GAsyncResult *result,
GError **error)
{
g_autoptr(GPtrArray) results = NULL;
g_return_val_if_fail (OSTREE_IS_REPO (self), NULL);
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_find_remotes_async), NULL);
g_return_val_if_fail (error == NULL || *error == NULL, NULL);
results = g_task_propagate_pointer (G_TASK (result), error);
g_assert (results == NULL);
return NULL;
}
void
ostree_repo_pull_from_remotes_async (OstreeRepo *self,
const OstreeRepoFinderResult * const *results,
GVariant *options,
OstreeAsyncProgress *progress,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
g_return_if_fail (OSTREE_IS_REPO (self));
g_task_report_new_error (self, callback, user_data, ostree_repo_find_remotes_async,
G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree was built without libsoup or libcurl, and cannot fetch over HTTP");
}
gboolean
ostree_repo_pull_from_remotes_finish (OstreeRepo *self,
GAsyncResult *result,
GError **error)
{
gboolean success;
g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_pull_from_remotes_async), FALSE);
g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
success = g_task_propagate_boolean (G_TASK (result), error);
g_assert (!success);
return FALSE;
}
#endif /* HAVE_LIBCURL_OR_LIBSOUP */