composetree: allow url in manifest package list

This allows URLs to be used in the manifest file when doing a
treecompose only in unified-core mode. This is a convenient way of
adding one off packages for testing purposes but should not be the
preferred method of composing a tree.

The import local rpm functions were also moved to rpmostree-importer
to facilitate usage.

src/daemon/rpmostreed-transaction-types.c

@@ -639,96 +639,6 @@ deploy_transaction_finalize (GObject *object)
   G_OBJECT_CLASS (deploy_transaction_parent_class)->finalize (object);
 }
 
-static gboolean
-import_local_rpm (OstreeRepo    *repo,
-                  int           *fd,
-                  char         **sha256_nevra,
-                  GCancellable  *cancellable,
-                  GError       **error)
-{
-  /* let's just use the current sepolicy -- we'll just relabel it if the new
-   * base turns out to have a different one */
-  glnx_autofd int rootfs_dfd = -1;
-  if (!glnx_opendirat (AT_FDCWD, "/", TRUE, &rootfs_dfd, error))
-    return FALSE;
-  g_autoptr(OstreeSePolicy) policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error);
-  if (policy == NULL)
-    return FALSE;
-
-  g_autoptr(RpmOstreeImporter) unpacker = rpmostree_importer_new_take_fd (fd, repo, NULL, 0, policy, error);
-  if (unpacker == NULL)
-    return FALSE;
-
-  if (!rpmostree_importer_run (unpacker, NULL, cancellable, error))
-    return FALSE;
-
-  g_autofree char *nevra = rpmostree_importer_get_nevra (unpacker);
-  *sha256_nevra = g_strconcat (rpmostree_importer_get_header_sha256 (unpacker),
-                               ":", nevra, NULL);
-
-  return TRUE;
-}
-
-static void
-ptr_close_fd (gpointer fdp)
-{
-  int fd = GPOINTER_TO_INT (fdp);
-  glnx_close_fd (&fd);
-}
-
-/* GUnixFDList doesn't allow stealing individual members */
-static GPtrArray *
-unixfdlist_to_ptrarray (GUnixFDList *fdl)
-{
-  gint len;
-  gint *fds = g_unix_fd_list_steal_fds (fdl, &len);
-  GPtrArray *ret = g_ptr_array_new_with_free_func ((GDestroyNotify)ptr_close_fd);
-  for (int i = 0; i < len; i++)
-    g_ptr_array_add (ret, GINT_TO_POINTER (fds[i]));
-  return ret;
-}
-
-static gboolean
-import_many_local_rpms (OstreeRepo    *repo,
-                        GUnixFDList   *fdl,
-                        GPtrArray    **out_pkgs,
-                        GCancellable  *cancellable,
-                        GError       **error)
-{
-  /* Note that we record the SHA-256 of the RPM header in the origin to make sure that e.g.
-   * if we somehow re-import the same NEVRA with different content, we error out. We don't
-   * record the checksum of the branch itself, because it may need relabeling and that's OK.
-   * */
-
-  g_auto(RpmOstreeRepoAutoTransaction) txn = { 0, };
-  /* Note use of commit-on-failure */
-  if (!rpmostree_repo_auto_transaction_start (&txn, repo, TRUE, cancellable, error))
-    return FALSE;
-
-  g_autoptr(GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_free);
-
-  g_autoptr(GPtrArray) fds = unixfdlist_to_ptrarray (fdl);
-  for (guint i = 0; i < fds->len; i++)
-    {
-      /* Steal fd from the ptrarray */
-      glnx_autofd int fd = GPOINTER_TO_INT (fds->pdata[i]);
-      fds->pdata[i] = GINT_TO_POINTER (-1);
-      g_autofree char *sha256_nevra = NULL;
-      /* Transfer fd to import */
-      if (!import_local_rpm (repo, &fd, &sha256_nevra, cancellable, error))
-        return FALSE;
-
-      g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra));
-    }
-
-  if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
-    return FALSE;
-  txn.initialized = FALSE;
-
-  *out_pkgs = g_steal_pointer (&pkgs);
-  return TRUE;
-}
-
 static void
 gv_nevra_add_nevra_name_mappings (GVariant *gv_nevra,
                                   GHashTable *name_to_nevra,
@@ -1079,7 +989,7 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
   if (self->install_local_pkgs != NULL)
     {
       g_autoptr(GPtrArray) pkgs = NULL;
-      if (!import_many_local_rpms (repo, self->install_local_pkgs, &pkgs,
+      if (!rpmostree_importer_import_many_local_rpms (repo, self->install_local_pkgs, &pkgs,
                                    cancellable, error))
         return FALSE;
 
@@ -1184,7 +1094,7 @@ deploy_transaction_execute (RpmostreedTransaction *transaction,
   if (self->override_replace_local_pkgs)
     {
       g_autoptr(GPtrArray) pkgs = NULL;
-      if (!import_many_local_rpms (repo, self->override_replace_local_pkgs, &pkgs,
+      if (!rpmostree_importer_import_many_local_rpms (repo, self->override_replace_local_pkgs, &pkgs,
                                    cancellable, error))
         return FALSE;
 

src/libpriv/rpmostree-core.c

@@ -1930,7 +1930,57 @@ rpmostree_context_prepare (RpmOstreeContext *self,
   g_autoptr(GHashTable) local_pkgs_to_install =
     g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_object_unref);
 
-  /* Handle packages to replace; only add them to the sack for now */
+  /* We download URI packages and import them into the pkgcache first so we can
+  add it to the local packages to install */
+  g_autoptr(GUnixFDList) fd_list = g_unix_fd_list_new ();
+  for (char **it = pkgnames; it && *it; it++)
+    {
+      const char *pkgname = *it;
+      g_assert (!self->rojig_pure);
+      if (g_str_has_prefix (*it, "http://") ||
+          g_str_has_prefix (*it, "https://"))
+        {
+          if (!self->pkgcache_repo)
+            {
+              g_print ("URLs are only supported in unified core mode\n");
+              return FALSE;
+            }
+
+          g_print ("Downloading package %s\n", pkgname);
+          glnx_autofd int fd = ror_download_to_fd (pkgname, error);
+          if (fd < 0)
+            return FALSE;
+
+          int idx = g_unix_fd_list_append (fd_list, fd, error);
+          if (idx < 0)
+            return FALSE;
+        }
+    }
+
+  if (g_unix_fd_list_get_length (fd_list) > 0)
+    {
+      OstreeRepo *repo = get_pkgcache_repo (self);
+      g_autoptr(GPtrArray) imported_pkgs = g_ptr_array_new ();
+      if (!rpmostree_importer_import_many_local_rpms (repo, fd_list, &imported_pkgs, cancellable, error))
+        return FALSE;
+
+      for (gsize i = 0; i < imported_pkgs->len; i++)
+        {
+          const char *nevra = imported_pkgs->pdata[i];
+          g_autofree char *sha256 = NULL;
+          if (!rpmostree_decompose_sha256_nevra (&nevra, &sha256, error))
+            return FALSE;
+
+          g_autoptr(DnfPackage) pkg = NULL;
+          if (!add_pkg_from_cache (self, nevra, sha256, &pkg, cancellable, error))
+            return FALSE;
+
+          g_hash_table_insert (local_pkgs_to_install, (gpointer)nevra, g_steal_pointer (&pkg));
+        }
+    }
+
+
+  /* Handle packages to replace */
   g_autoptr(GPtrArray) replaced_nevras = g_ptr_array_new ();
   for (char **it = cached_replace_pkgs; it && *it; it++)
     {
@@ -2024,19 +2074,23 @@ rpmostree_context_prepare (RpmOstreeContext *self,
     {
       const char *pkgname = *it;
       g_assert (!self->rojig_pure);
-      g_autoptr(GError) local_error = NULL;
-      if (!dnf_context_install (dnfctx, pkgname, &local_error))
+      if (!g_str_has_prefix (*it, "http://") &&
+          !g_str_has_prefix (*it, "https://"))
         {
-          /* Only keep going if it's ENOENT, so we coalesce into one msg at the end */
-          if (!g_error_matches (local_error, DNF_ERROR, DNF_ERROR_PACKAGE_NOT_FOUND))
+          g_autoptr(GError) local_error = NULL;
+          if (!dnf_context_install (dnfctx, pkgname, &local_error))
             {
-              g_propagate_error (error, g_steal_pointer (&local_error));
-              return FALSE;
+              /* Only keep going if it's ENOENT, so we coalesce into one msg at the end */
+              if (!g_error_matches (local_error, DNF_ERROR, DNF_ERROR_PACKAGE_NOT_FOUND))
+                {
+                  g_propagate_error (error, g_steal_pointer (&local_error));
+                  return FALSE;
+                }
+              /* lazy init since it's unlikely in the common case (e.g. upgrades) */
+              if (!missing_pkgs)
+                missing_pkgs = g_ptr_array_new ();
+              g_ptr_array_add (missing_pkgs, (gpointer)pkgname);
             }
-          /* lazy init since it's unlikely in the common case (e.g. upgrades) */
-          if (!missing_pkgs)
-            missing_pkgs = g_ptr_array_new ();
-          g_ptr_array_add (missing_pkgs, (gpointer)pkgname);
         }
     }
 
@@ -3799,7 +3853,7 @@ process_ostree_layers (RpmOstreeContext *self,
 {
   if (!self->treefile_rs)
     return TRUE;
-  
+
   g_auto(GStrv) layers = ror_treefile_get_ostree_layers (self->treefile_rs);
   g_auto(GStrv) override_layers = ror_treefile_get_ostree_override_layers (self->treefile_rs);
   const size_t n = (layers ? g_strv_length (layers) : 0) + (override_layers ? g_strv_length (override_layers) : 0);

src/libpriv/rpmostree-core.h

@@ -21,6 +21,7 @@
 #pragma once
 
 #include <gio/gio.h>
+#include <gio/gunixfdlist.h>
 #include <libdnf/libdnf.h>
 #include <ostree.h>
 

src/libpriv/rpmostree-importer.c

@@ -1030,3 +1030,94 @@ rpmostree_importer_get_header_sha256 (RpmOstreeImporter *self)
 {
   return self->hdr_sha256;
 }
+
+
+static gboolean
+import_local_rpm (OstreeRepo    *repo,
+                  int           *fd,
+                  char         **sha256_nevra,
+                  GCancellable  *cancellable,
+                  GError       **error)
+{
+  /* let's just use the current sepolicy -- we'll just relabel it if the new
+   * base turns out to have a different one */
+  glnx_autofd int rootfs_dfd = -1;
+  if (!glnx_opendirat (AT_FDCWD, "/", TRUE, &rootfs_dfd, error))
+    return FALSE;
+  g_autoptr(OstreeSePolicy) policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error);
+  if (policy == NULL)
+    return FALSE;
+
+  g_autoptr(RpmOstreeImporter) unpacker = rpmostree_importer_new_take_fd (fd, repo, NULL, 0, policy, error);
+  if (unpacker == NULL)
+    return FALSE;
+
+  if (!rpmostree_importer_run (unpacker, NULL, cancellable, error))
+    return FALSE;
+
+  g_autofree char *nevra = rpmostree_importer_get_nevra (unpacker);
+  *sha256_nevra = g_strconcat (rpmostree_importer_get_header_sha256 (unpacker),
+                               ":", nevra, NULL);
+
+  return TRUE;
+}
+
+static void
+ptr_close_fd (gpointer fdp)
+{
+  int fd = GPOINTER_TO_INT (fdp);
+  glnx_close_fd (&fd);
+}
+
+/* GUnixFDList doesn't allow stealing individual members */
+static GPtrArray *
+unixfdlist_to_ptrarray (GUnixFDList *fdl)
+{
+  gint len;
+  gint *fds = g_unix_fd_list_steal_fds (fdl, &len);
+  GPtrArray *ret = g_ptr_array_new_with_free_func ((GDestroyNotify)ptr_close_fd);
+  for (int i = 0; i < len; i++)
+    g_ptr_array_add (ret, GINT_TO_POINTER (fds[i]));
+  return ret;
+}
+
+gboolean
+rpmostree_importer_import_many_local_rpms (OstreeRepo    *repo,
+                        GUnixFDList   *fdl,
+                        GPtrArray    **out_pkgs,
+                        GCancellable  *cancellable,
+                        GError       **error)
+{
+  /* Note that we record the SHA-256 of the RPM header in the origin to make sure that e.g.
+   * if we somehow re-import the same NEVRA with different content, we error out. We don't
+   * record the checksum of the branch itself, because it may need relabeling and that's OK.
+   * */
+
+  g_auto(RpmOstreeRepoAutoTransaction) txn = { 0, };
+  /* Note use of commit-on-failure */
+  if (!rpmostree_repo_auto_transaction_start (&txn, repo, TRUE, cancellable, error))
+    return FALSE;
+
+  g_autoptr(GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_free);
+
+  g_autoptr(GPtrArray) fds = unixfdlist_to_ptrarray (fdl);
+  for (guint i = 0; i < fds->len; i++)
+    {
+      /* Steal fd from the ptrarray */
+      glnx_autofd int fd = GPOINTER_TO_INT (fds->pdata[i]);
+      fds->pdata[i] = GINT_TO_POINTER (-1);
+      g_autofree char *sha256_nevra = NULL;
+      /* Transfer fd to import */
+      if (!import_local_rpm (repo, &fd, &sha256_nevra, cancellable, error))
+        return FALSE;
+
+      g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra));
+    }
+
+  if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
+    return FALSE;
+  txn.initialized = FALSE;
+
+  *out_pkgs = g_steal_pointer (&pkgs);
+  return TRUE;
+}

src/libpriv/rpmostree-importer.h

@@ -90,3 +90,11 @@ rpmostree_importer_get_nevra (RpmOstreeImporter *self);
 
 const char *
 rpmostree_importer_get_header_sha256 (RpmOstreeImporter *self);
+
+gboolean
+rpmostree_importer_import_many_local_rpms (OstreeRepo    *repo,
+
+                        GUnixFDList   *fdl,
+                        GPtrArray    **out_pkgs,
+                        GCancellable  *cancellable,
+                        GError       **error);