#!/usr/bin/python3 # # eos-update-flatpak-repos: adds standard flatpak repos, removes legacy remotes # and runtimes, and migrates installed flatpaks between origins # # Copyright © 2017–2020 Endless OS LLC # Authors: # Mario Sanchez Prada # Philip Chimento # Robert McQueen # Will Thompson # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import argparse import fnmatch import logging import os import subprocess import sys from urllib.parse import urlparse, urlunparse from systemd import journal import gi gi.require_version('Flatpak', '1.0') gi.require_version('OSTree', '1.0') from gi.repository import Flatpak, Gio, GLib, OSTree # noqa: E402 def _flatpak_inst_get_remote(inst, name): try: remote = inst.get_remote_by_name(name) except GLib.Error: remote = None return remote FLATPAK_REPO_DIR = os.getenv('EOS_FLATPAK_REPO_DIR', '/usr/share/eos-boot-helper/flatpak-repos') def _add_flatpak_repos(): insts = Flatpak.get_system_installations() inst = insts[0] for f in os.listdir(FLATPAK_REPO_DIR): if not f.endswith('.flatpakrepo'): continue name = f[:-12] repo_file = os.path.join(FLATPAK_REPO_DIR, f) if not os.path.isfile(repo_file): continue remote = _flatpak_inst_get_remote(inst, name) if remote: logging.debug("Remote {} already configured in {}" .format(name, inst.get_path().get_path())) continue logging.info("Adding remote {} to {} from {}" .format(name, inst.get_path().get_path(), repo_file)) subprocess.check_call(['flatpak', 'remote-add', '--system', '--from', name, repo_file]) # list of ostree refs OSTREE_REFS_TO_REMOVE = [ # remove eos3.3 refs leaked by eos-updater checkpoint bug (T23410) 'eos:os/eos/amd64/eos3', 'eos:os/eos/ec100/eos3', 'eos:os/eos/nexthw/eos3' # remove eos2 ref left behind after eos-upgrade-eos2-to-eos3 (T23443) 'eos:eos2/ec100', 'eos:eos2/i386', # remove "initial factory commit" ref from pre-2.3.1 (T23443) 'local:factory', ] def _remove_ostree_refs(): try: repo = OSTree.Repo.new_default() repo.open() _, ostree_refs = repo.list_refs() # dictionary of ref -> commit for ref in OSTREE_REFS_TO_REMOVE: if ref in ostree_refs: logging.info('Deleting leaked OSTree ref {}'.format(ref)) repo.set_ref_immediate(ref.split(':')[0], ref.split(':')[1], None) except Exception: logging.exception('Failure deleting leaked OSTree refs') # list of remote names REMOTES_TO_REMOVE = [ ] def _remove_remotes(): insts = Flatpak.get_system_installations() inst = insts[0] for name in REMOTES_TO_REMOVE: if _flatpak_inst_get_remote(inst, name): logging.info("Removing remote {} from {}" .format(name, inst.get_path().get_path())) subprocess.check_call(['flatpak', 'remote-delete', '--system', '--force', name]) # dict of remote -> list of runtimes RUNTIMES_TO_REMOVE = { } def _remove_runtimes(): insts = Flatpak.get_system_installations() inst = insts[0] for ref in inst.list_installed_refs_by_kind(Flatpak.RefKind.RUNTIME): origin = ref.get_origin() if ref.get_origin() not in RUNTIMES_TO_REMOVE: continue name = ref.get_name() if name not in RUNTIMES_TO_REMOVE[origin]: continue refspec = '{}/{}/{}'.format(name, ref.get_arch(), ref.get_branch()) logging.info("Removing runtime {} from {}" .format(refspec, inst.get_path().get_path())) subprocess.check_call(['flatpak', 'uninstall', '--system', '--runtime', refspec]) # From 3.7.6 through 3.7.8, images were shipped with remote URLs using # our internal ostree.endlessm-sf.com domain instead of our # ostree.endlessm.com CDN domain. # # https://phabricator.endlessm.com/T29535 def _fix_url(url): parts = urlparse(url) if parts.netloc == 'ostree.endlessm-sf.com': parts = parts._replace(netloc='ostree.endlessm.com') url = urlunparse(parts) return url def _fix_remote_urls(): _, repo = OSTree.Sysroot().get_repo() repo.open() for remote in repo.remote_list(): # Only adjust non-flatpak remotes (xa.disable = true) _, is_os_repo = repo.get_remote_boolean_option(remote, 'xa.disable', False) if not is_os_repo: logging.debug('Skipping non-OSTree remote %s', remote) continue _, cur_url = repo.remote_get_url(remote) logging.debug('OSTree remote %s current URL %s', remote, cur_url) new_url = _fix_url(cur_url) if new_url != cur_url: # OSTree offers no reasonable API for this logging.info('Changing OSTree remote %s URL from %s to %s', remote, cur_url, new_url) config = repo.copy_config() remote_group = 'remote "{}"'.format(remote) config.set_string(remote_group, 'url', new_url) repo.write_config(config) for inst in Flatpak.get_system_installations(): for remote in inst.list_remotes(): cur_url = remote.get_url() logging.debug('Flatpak remote %s current URL %s', remote.get_name(), cur_url) new_url = _fix_url(cur_url) if new_url != cur_url: logging.info('Changing Flatpak remote %s URL from %s to %s', remote.get_name(), cur_url, new_url) remote.set_url(new_url) inst.modify_remote(remote) def update_deploy_file_with_origin(deploy_file_path, new_origin): orig_variant = load_deploy_file(deploy_file_path) if orig_variant[0] == new_origin: logging.info('Nothing to do, {} origin already set to: {}' .format(deploy_file_path, new_origin)) return write_deploy_file(deploy_file_path, orig_variant, 0, GLib.Variant('s', new_origin)) def update_deploy_file_with_previous_ids(deploy_file_path, new_previous_ids): orig_variant = load_deploy_file(deploy_file_path) metadata = GLib.VariantDict(orig_variant.get_child_value(4)) previous_ids_variant = metadata.lookup_value('previous-ids') if previous_ids_variant: previous_ids = set(previous_ids_variant.unpack()) else: previous_ids = set() to_add = new_previous_ids - previous_ids if not to_add: logging.info('Nothing to do, {} already contains previous-id: {}' .format(deploy_file_path, new_previous_ids)) return logging.debug('Adding {} to previous-ids'.format(to_add)) previous_ids.update(new_previous_ids) metadata.insert_value('previous-ids', GLib.Variant('as', list(previous_ids))) write_deploy_file(deploy_file_path, orig_variant, 4, metadata.end()) def load_deploy_file(deploy_file_path): logging.debug("Reading data from the deploy file at {}..." .format(deploy_file_path)) src_file_contents = None with open(deploy_file_path, 'rb') as f: src_file_contents = GLib.Bytes.new(f.read()) # We need to read the GVariant in the deploy file and generate a # new one with all the same content but the right remote set. variant_type = GLib.VariantType.new('(ssasta{sv})') orig_variant = GLib.Variant.new_from_bytes(variant_type, src_file_contents, False) logging.debug("Original variant: {}".format(str(orig_variant))) return orig_variant def write_deploy_file(deploy_file_path, orig_variant, idx, new_val): variant_type = GLib.VariantType.new('(ssasta{sv})') builder = GLib.VariantBuilder.new(variant_type) for i, val in enumerate(orig_variant): child = orig_variant.get_child_value(i) if i == idx: builder.add_value(new_val) else: builder.add_value(child) new_variant = builder.end() logging.debug("New variant: {}".format(str(new_variant))) # Replace the original deploy file with the new GVariant logging.info("Writing new deploy file for {}, setting [{}] to: {}" .format(deploy_file_path, idx, new_val)) GLib.file_set_contents(deploy_file_path, new_variant.get_data_as_bytes().get_data()) def mangle_firefox_metadata(metadata): """Endless's Firefox Flatpak stored its data in the real ~/.mozilla in the user's home directory. Mozilla's Firefox Flatpak stores its data in what it thinks is ~/.mozilla, but which is actually ~/.var/app/$APP_ID/.mozilla through the magic of --persist=.mozilla. Monkeypatch our Flatpak to match. eos-migrate-firefox-profile is run via a systemd user unit to actually move the data.""" filesystems = _get_with_default( metadata.get_string_list, FLATPAK_METADATA_GROUP_CONTEXT, FLATPAK_METADATA_FILESYSTEMS, [], ) if "home" in filesystems: filesystems.remove("home") if "xdg-download" not in filesystems: filesystems.append("xdg-download") metadata.set_string_list( FLATPAK_METADATA_GROUP_CONTEXT, FLATPAK_METADATA_FILESYSTEMS, filesystems, ) persistent = _get_with_default( metadata.get_string_list, FLATPAK_METADATA_GROUP_CONTEXT, FLATPAK_METADATA_PERSISTENT, [], ) if ".mozilla" not in persistent: persistent.append(".mozilla") metadata.set_string_list( FLATPAK_METADATA_GROUP_CONTEXT, FLATPAK_METADATA_PERSISTENT, persistent, ) FLATPAKS_TO_MIGRATE = [ # migrate EknServices from old eos-apps remote to eos-sdk (T17863) { 'name': 'com.endlessm.EknServices', 'kind': Flatpak.RefKind.APP, 'old-origin': 'eos-apps', 'new-origin': 'eos-sdk' }, # fix up image-builder bug where com.endlessm.apps.* runtimes were # mistakenly installed with eos-runtimes origin (T18366) { 'name': 'com.endlessm.apps.Platform', 'kind': Flatpak.RefKind.RUNTIME, 'old-origin': 'eos-runtimes', 'new-origin': 'eos-sdk' }, { 'name': 'com.endlessm.apps.Sdk', 'kind': Flatpak.RefKind.RUNTIME, 'old-origin': 'eos-runtimes', 'new-origin': 'eos-sdk' }, # migrate Steam from eos-apps to flathub (T20322) { 'name': 'com.valvesoftware.Steam', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate org.gnome.* from gnome-apps to flathub (T19898) { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.APP, 'old-branch': 'stable', 'old-origin': 'gnome-apps', 'new-origin': 'flathub' }, # migrate Teeworlds and MegaGlest from eos-apps to flathub (T20411) { 'name': 'com.teeworlds.Teeworlds', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.megaglest.MegaGlest', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Atom from eos-apps to flathub (T20301) { 'name': 'io.atom.Atom', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate gnome 3.2[02468] and freedesktop 1.[46] runtimes to flathub # (T20443 and T25809 and T31079) { 'prefix': 'org.freedesktop.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '1.4', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.freedesktop.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '1.6', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '3.20', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '3.22', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '3.24', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '3.26', 'old-origin': 'gnome', 'new-origin': 'flathub' }, { 'prefix': 'org.gnome.', 'kind': Flatpak.RefKind.RUNTIME, 'old-branch': '3.28', 'old-origin': 'gnome', 'new-origin': 'flathub' }, # migrate eos-apps to flathub where the app ID is the same (T20724) { 'name': 'com.google.AndroidStudio', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'com.slack.Slack', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'com.spotify.Client', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'com.transmissionbt.Transmission', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.minetest.Minetest', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.freeciv.Freeciv', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Builder', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Genius', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Gnote', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.inkscape.Inkscape', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.pitivi.Pitivi', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.tuxpaint.Tuxpaint', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.videolan.VLC', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.wesnoth.Wesnoth', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Visual Studio Code OSS from eos-apps to flathub (T20306) { 'name': 'com.visualstudio.code.oss', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Sublime Text from eos-apps to flathub (T22875) { 'name': 'com.sublimetext.three', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Photo Editor from eos-apps to flathub (T21891) { 'name': 'com.endlessm.photos', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Dropbox from eos-apps to flathub (T23174) { 'name': 'com.dropbox.Client', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Stellarium from eos-apps to flathub (T23681) { 'name': 'org.stellarium.Stellarium', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate 61 legacy eos-apps to flathub (T23856) { 'name': 'com.arduino.App', 'new-name': 'cc.arduino.arduinoide', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'com.github.Slingshot', 'new-name': 'com.github.ryanakca.slingshot', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'io.github.Supertux', 'new-name': 'org.supertuxproject.SuperTux', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.blockout.Blockout2', 'new-name': 'net.blockout.BlockOutII', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.gcompris.Gcompris', 'new-name': 'org.kde.gcompris', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.olofson.Kobodeluxe', 'new-name': 'net.olofson.KoboDeluxe', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Atanks', 'new-name': 'net.sourceforge.atanks', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Audacity', 'new-name': 'org.audacityteam.Audacity', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Btanks', 'new-name': 'net.sourceforge.btanks', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.ChromiumBSU', 'new-name': 'net.sourceforge.chromium-bsu', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Extremetuxracer', 'new-name': 'net.sourceforge.ExtremeTuxRacer', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Rili', 'new-name': 'net.sourceforge.Ri-li', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Supertuxkart', 'new-name': 'net.supertuxkart.SuperTuxKart', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Torcs', 'new-name': 'net.sourceforge.torcs', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Tuxfootball', 'new-name': 'net.sourceforge.TuxFootball', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.sourceforge.Warmux', 'new-name': 'org.gna.Warmux', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'net.wz2100.Warzone2100', 'new-name': 'net.wz2100.wz2100', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.armagetronad.Armagetronad', 'new-name': 'org.armagetronad.ArmagetronAdvanced', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.codeblocks.App', 'new-name': 'org.codeblocks.codeblocks', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.debian.Tuxpuck', 'new-name': 'org.debian.TuxPuck', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.debian.alioth.tux4kids.Tuxmath', 'new-name': 'com.tux4kids.tuxmath', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.debian.alioth.tux4kids.Tuxtype', 'new-name': 'com.tux4kids.tuxtype', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.frozenbubble.FrozenBubble', 'new-name': 'org.frozen_bubble.frozen-bubble', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gimp.Gimp', 'new-name': 'org.gimp.GIMP', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Iagno', 'new-name': 'org.gnome.iagno', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Quadrapassel', 'new-name': 'org.gnome.quadrapassel', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.Tetravex', 'new-name': 'org.gnome.tetravex', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.gnome.people.dscorgie.Labyrinth', 'new-name': 'com.github.labyrinth_team.labyrinth', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kalzium', 'new-name': 'org.kde.kalzium', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kapman', 'new-name': 'org.kde.kapman', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Katomic', 'new-name': 'org.kde.katomic', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kblocks', 'new-name': 'org.kde.kblocks', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kbounce', 'new-name': 'org.kde.kbounce', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kbruch', 'new-name': 'org.kde.kbruch', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kdiamond', 'new-name': 'org.kde.kdiamond', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kgeography', 'new-name': 'org.kde.kgeography', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kgoldrunner', 'new-name': 'org.kde.kgoldrunner', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Khangman', 'new-name': 'org.kde.khangman', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kigo', 'new-name': 'org.kde.kigo', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Killbots', 'new-name': 'org.kde.killbots', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kjumpingcube', 'new-name': 'org.kde.kjumpingcube', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Klines', 'new-name': 'org.kde.klines', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Knavalbattle', 'new-name': 'org.kde.knavalbattle', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Knetwalk', 'new-name': 'org.kde.knetwalk', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Ksame', 'new-name': 'org.kde.klickety', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Ksquares', 'new-name': 'org.kde.ksquares', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Ksudoku', 'new-name': 'org.kde.ksudoku', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Ktuberling', 'new-name': 'org.kde.ktuberling', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kubrick', 'new-name': 'org.kde.kubrick', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Kwordquiz', 'new-name': 'org.kde.kwordquiz', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.kde.Palapeli', 'new-name': 'org.kde.palapeli', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.maemo.Numptyphysics', 'new-name': 'io.thp.numptyphysics', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.marsshooter.Marsshooter', 'new-name': 'net.sourceforge.mars-game', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.openarena.Openarena', 'new-name': 'ws.openarena.OpenArena', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.openscad.Openscad', 'new-name': 'org.openscad.OpenSCAD', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.processing.App', 'new-name': 'org.processing.processingide', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.seul.Pingus', 'new-name': 'org.seul.pingus', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.sugarlabs.Turtleblocks', 'new-name': 'org.laptop.TurtleArtActivity', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'org.tuxfamily.Xmoto', 'new-name': 'org.tuxfamily.XMoto', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Skype to flathub (T24741) { 'name': 'com.microsoft.Skype', 'new-name': 'com.skype.Client', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Minecraft to flathub (T26588) { 'name': 'com.mojang.Minecraft', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Scratch to flathub (T26658) { 'name': 'org.squeakland.Scratch', 'new-name': 'edu.mit.Scratch', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate endlessnetwork apps to flathub (T25928) { 'name': 'com.endlessm.MidnightmareTeddy', 'new-name': 'com.endlessnetwork.MidnightmareTeddy', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, { 'name': 'com.endlessm.MissileMath', 'new-name': 'com.endlessnetwork.missilemath', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate BillardGL from eos-apps to flathub (T27078) { 'name': 'de.billardgl.Billardgl', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Etoys from eos-apps to flathub (T28386) { 'name': 'org.squeakland.Etoys', 'kind': Flatpak.RefKind.APP, 'old-branch': 'eos3', 'new-branch': 'stable', 'old-origin': 'eos-apps', 'new-origin': 'flathub' }, # migrate Eclipse from eos-apps to flathub (T28707) { "name": "org.eclipse.Eclipse", "new-name": "org.eclipse.Java", "kind": Flatpak.RefKind.APP, "old-branch": "eos3", "new-branch": "stable", "old-origin": "eos-apps", "new-origin": "flathub", }, # migrate Firefox from eos-apps to flathub (T29746) { "name": "org.mozilla.Firefox", "new-name": "org.mozilla.firefox", "kind": Flatpak.RefKind.APP, "old-branch": "eos3", "new-branch": "stable", "old-origin": "eos-apps", "new-origin": "flathub", "mangle-metadata": mangle_firefox_metadata, }, ] def _filter_refs(refs, name=None, prefix=None, kind=None, arch=None, branch=None, origin=None): ret = [] for ref in refs: if kind and kind != ref.get_kind(): continue if name and name != ref.get_name(): continue if prefix and not ref.get_name().startswith(prefix): continue if arch and arch != ref.get_arch(): continue if branch and branch != ref.get_branch(): continue if origin and origin != ref.get_origin(): continue ret.append(ref) return ret def _filter_ostree_refs(refs, origin, kind, name='*', prefix=None, arch='*', branch='*'): assert origin assert kind.value_nick in {'app', 'runtime'} if prefix: name = '{}*'.format(prefix) pattern = '{}:{}/{}/{}/{}'.format(origin, kind.value_nick, name, arch, branch) return fnmatch.filter(refs, pattern) def mtree_add_file(repo, mtree, stream, file_info): file_info.set_file_type(Gio.FileType.REGULAR) file_info.set_attribute_uint32('unix::uid', 0) file_info.set_attribute_uint32('unix::gid', 0) file_info.set_attribute_uint32('unix::mode', 0o100644) _, content_stream, content_length = OSTree.raw_file_to_content_stream( stream, file_info, None ) _, csum_bytes = repo.write_content(None, content_stream, content_length) csum = OSTree.checksum_from_bytes(csum_bytes) mtree.replace_file(file_info.get_name(), csum) def mtree_load_file(repo, mtree, name): files = mtree.get_files() csum = files[name] _, stream, info, _ = repo.load_file(csum) info.set_name(name) return stream, info def mtree_rename_file(repo, mtree, old_name, new_name): stream, info = mtree_load_file(repo, mtree, old_name) info.set_name(new_name) mtree_add_file(repo, mtree, stream, info) mtree.remove(old_name, False) def mtree_load_keyfile(repo, mtree, name): stream, info = mtree_load_file(repo, mtree, name) data = stream.read_bytes(info.get_size()) keyfile = GLib.KeyFile() keyfile.load_from_bytes(data, GLib.KeyFileFlags.KEEP_COMMENTS | GLib.KeyFileFlags.KEEP_TRANSLATIONS) return keyfile, info def mtree_add_keyfile(repo, mtree, keyfile, info): keyfile_string = keyfile.to_data()[0] keyfile_bytes = keyfile_string.encode('utf-8') stream = Gio.MemoryInputStream.new_from_data(keyfile_bytes) info.set_size(len(keyfile_bytes)) mtree_add_file(repo, mtree, stream, info) return keyfile_string def _get_with_default(func, group, key, default): try: return func(group, key) except GLib.GError as e: if not e.matches(GLib.KeyFile.error_quark(), GLib.KeyFileError.KEY_NOT_FOUND): raise return default def mangle_desktop_file(desktop_file, old_name, old_id, new_id): # if the app relied on D-Bus activation, the desktop file ID # will no longer match the name that is implemented internally # so we have to disable it da = _get_with_default( desktop_file.get_boolean, GLib.KEY_FILE_DESKTOP_GROUP, GLib.KEY_FILE_DESKTOP_KEY_DBUS_ACTIVATABLE, False ) if da: logging.debug('Removing DBusActivatable=true') desktop_file.remove_key( GLib.KEY_FILE_DESKTOP_GROUP, GLib.KEY_FILE_DESKTOP_KEY_DBUS_ACTIVATABLE ) # update the icon name which matches the ID value = _get_with_default( desktop_file.get_string, GLib.KEY_FILE_DESKTOP_GROUP, GLib.KEY_FILE_DESKTOP_KEY_ICON, None ) if value is not None: value = value.replace(old_id, new_id) desktop_file.set_string( GLib.KEY_FILE_DESKTOP_GROUP, GLib.KEY_FILE_DESKTOP_KEY_ICON, value ) # Add old name to X-Flatpak-RenamedFrom renamed_from = _get_with_default( desktop_file.get_string_list, GLib.KEY_FILE_DESKTOP_GROUP, 'X-Flatpak-RenamedFrom', [] ) if old_name not in renamed_from: renamed_from.append(old_name) desktop_file.set_string_list(GLib.KEY_FILE_DESKTOP_GROUP, 'X-Flatpak-RenamedFrom', renamed_from) def rename_exports(repo, mtree, old_id, new_id, parent_mtree=None, path='export'): logging.debug('Renaming files in {}'.format(path)) vendor_prefixes = set() files = mtree.get_files() for name, csum in files.items(): if name.startswith(old_id): new_name = new_id + name[len(old_id):] if path == 'export/share/applications' and \ name.endswith('.desktop'): logging.info('Rewriting {} as {}'.format(name, new_name)) desktop_file, info = mtree_load_keyfile(repo, mtree, name) mangle_desktop_file(desktop_file, name, old_id, new_id) info.set_name(new_name) mtree_add_keyfile(repo, mtree, desktop_file, info) mtree.remove(name, False) elif ( os.path.dirname(path) == 'export/share/applications' and name.endswith('.desktop') ): # Vendor subdirectory, as used by kde4-... apps vendor = os.path.basename(path) vendor_prefixes.add(vendor) logging.info('Rewriting {}/{} as {}'.format(vendor, name, new_name)) desktop_file, info = mtree_load_keyfile(repo, mtree, name) # kde4/foo.desktop is treated as kde4-foo.desktop, so put that as the # old name into the rewritten file. old_name = vendor + '-' + name mangle_desktop_file(desktop_file, old_name, old_id, new_id) info.set_name(new_name) mtree_add_keyfile(repo, parent_mtree, desktop_file, info) mtree.remove(name, False) elif path == 'export/share/dbus-1/services' and name.endswith('.service'): logging.info('Not changing exported D-Bus service {}'.format(name)) else: logging.info('Renaming {} to {}'.format(name, new_name)) mtree_rename_file(repo, mtree, name, new_name) else: logging.debug('Not changing {}'.format(name)) dirs = mtree.get_subdirs() for k, v in dirs.items(): vendor_prefixes.update( rename_exports(repo, v, old_id, new_id, mtree, os.path.join(path, k)) ) return vendor_prefixes FLATPAK_METADATA_FILE = 'metadata' FLATPAK_METADATA_APPLICATION = 'Application' FLATPAK_METADATA_KEY_NAME = 'name' FLATPAK_METADATA_GROUP_SESSION_BUS_POLICY = 'Session Bus Policy' FLATPAK_METADATA_BUS_POLICY_OWN = 'own' FLATPAK_METADATA_GROUP_EXTRA_DATA = 'Extra Data' FLATPAK_METADATA_GROUP_CONTEXT = "Context" FLATPAK_METADATA_FILESYSTEMS = "filesystems" FLATPAK_METADATA_PERSISTENT = "persistent" def rewrite_metadata(repo, mtree, old_id, new_id, mangle_metadata=None): metadata, info = mtree_load_keyfile(repo, mtree, FLATPAK_METADATA_FILE) if old_id != new_id: metadata.set_string(FLATPAK_METADATA_APPLICATION, FLATPAK_METADATA_KEY_NAME, new_id) # allow the app to own its old bus name (if the app was relying on this, # it would lose the right to this name with the new ID) metadata.set_string(FLATPAK_METADATA_GROUP_SESSION_BUS_POLICY, old_id, FLATPAK_METADATA_BUS_POLICY_OWN) # extra data apps are handled by removing the [Extra Data] from the metadata # of the migrated app to allow the deploy to be done offline. instead, # after the new app is installed, this script will copy the /app/extra # folder from the pre-migration deploy folder. the migration function # examines the commit metadata of the app being migrated to determine # whether the re-deploy has to carry out this extra step. try: metadata.remove_group(FLATPAK_METADATA_GROUP_EXTRA_DATA) logging.info('Removing [Extra Data] from {} metadata'.format(old_id)) except GLib.GError as e: if not e.matches(GLib.key_file_error_quark(), GLib.KeyFileError.GROUP_NOT_FOUND): raise if mangle_metadata is not None: mangle_metadata(metadata) metadata_str = mtree_add_keyfile(repo, mtree, metadata, info) if old_id == new_id: vendor_prefixes = frozenset() return metadata_str, vendor_prefixes dirs = mtree.get_subdirs() if 'export' in dirs: export_mtree = dirs['export'] vendor_prefixes = rename_exports(repo, export_mtree, old_id, new_id, mtree) else: logging.warning('Cannot find export dir in {} mtree' .format(old_id)) vendor_prefixes = frozenset() return metadata_str, vendor_prefixes COMMIT_SUBJECT_INDEX = 3 COMMIT_BODY_INDEX = 4 def copy_commit(repo, src_rev, dest_ref, old_app_id=None, new_app_id=None, collection_id=None, migrate_extra=False, mangle_metadata=None): """Copy commit src_rev to dest_ref This makes the new commit at dest_ref have the proper collection binding for this repo. The caller is expected to manage the ostree transaction. This is like "flatpak build-commit-from", but we need more control over the transaction. """ logging.debug('Copying commit %s to %s', src_rev, dest_ref) _, src_root, _ = repo.read_commit(src_rev) _, src_variant, src_state = repo.load_commit(src_rev) # Only copy normal commits if src_state != 0: raise Exception('Cannot copy irregular commit {}' .format(src_rev)) # If the dest ref exists, use the current commit as the new # commit's parent _, dest_parent = repo.resolve_rev_ext( dest_ref, allow_noent=True, flags=OSTree.RepoResolveRevExtFlags.NONE) if dest_parent is not None: logging.info('Using %s as new commit parent', dest_parent) # Make a copy of the commit metadata to update. Like flatpak # build-commit-from, the detached metadata is not copied since # the only known usage is for GPG signatures, which would become # invalid. commit_metadata = GLib.VariantDict.new(src_variant.get_child_value(0)) # Set the collection binding if the repo has a collection ID, # otherwise remove it if collection_id is not None: commit_metadata.insert_value( OSTree.COMMIT_META_KEY_COLLECTION_BINDING, GLib.Variant('s', collection_id)) else: commit_metadata.remove( OSTree.COMMIT_META_KEY_COLLECTION_BINDING) # Include the destination ref in the ref bindings ref_bindings = commit_metadata.lookup_value( OSTree.COMMIT_META_KEY_REF_BINDING, GLib.VariantType('as')) if ref_bindings is None: ref_bindings = [] ref_bindings = set(ref_bindings) ref_bindings.add(dest_ref) commit_metadata.insert_value( OSTree.COMMIT_META_KEY_REF_BINDING, GLib.Variant('as', sorted(ref_bindings))) # Add flatpak specific metadata. xa.ref is deprecated, but some # flatpak clients might expect it. xa.from_commit will be used # by the app verifier to make sure the commit it sent actually # got there commit_metadata.insert_value('xa.ref', GLib.Variant('s', dest_ref)) commit_metadata.insert_value('xa.from_commit', GLib.Variant('s', src_rev)) # Copy other commit data from source commit commit_subject = src_variant[COMMIT_SUBJECT_INDEX] commit_body = src_variant[COMMIT_BODY_INDEX] commit_time = OSTree.commit_get_timestamp(src_variant) # Make the new commit assuming the caller started a transaction mtree = OSTree.MutableTree.new() repo.write_directory_to_mtree(src_root, mtree, None) if old_app_id and new_app_id and \ (old_app_id != new_app_id or migrate_extra): metadata, vendor_prefixes = rewrite_metadata(repo, mtree, old_app_id, new_app_id, mangle_metadata) commit_metadata.insert_value('xa.metadata', GLib.Variant('s', metadata)) # unconditionally remove extra data, the caller determines whether the # the extra data dir needs migrating commit_metadata.remove('xa.extra-data-sources') else: vendor_prefixes = frozenset() # Convert from GVariantDict to GVariant vardict commit_metadata = commit_metadata.end() _, dest_root = repo.write_mtree(mtree) _, dest_checksum = repo.write_commit_with_time(dest_parent, commit_subject, commit_body, commit_metadata, dest_root, commit_time) logging.info('Created new commit %s', dest_checksum) return dest_checksum, vendor_prefixes def migrate_extra_data(old_ref, new_ref): old_deploy = old_ref.get_deploy_dir() old_extra = os.path.join(old_deploy, 'files', 'extra') new_deploy = new_ref.get_deploy_dir() new_extra = os.path.join(new_deploy, 'files', 'extra') if os.path.exists(new_extra): # either the user manually installed Skype from Flathub, or we got # this far but died before uninstalling the old ref last time logging.info('%s already has extra data dir', new_ref.format_ref()) else: logging.debug('Copying extra data from %s to %s', old_ref.format_ref(), new_ref.format_ref()) subprocess.check_call(['cp', '-alT', old_extra, new_extra + '.tmp']) os.rename(new_extra + '.tmp', new_extra) def is_extra_data(repo, rev): _, commit_variant, _ = repo.load_commit(rev) commit_metadata = GLib.VariantDict.new(commit_variant.get_child_value(0)) return commit_metadata.contains('xa.extra-data-sources') def _migrate_installed_flatpaks(): insts = Flatpak.get_system_installations() inst = insts[0] refs = inst.list_installed_refs() triggers_needed = False # we need to also operate directly on the ostree refs that shadow each # flatpak so we don't leave behind local refs to the old origin and branch. repo = OSTree.Repo.new(inst.get_path().get_child('repo')) repo.open() _, ostree_refs = repo.list_refs() # dictionary of ref -> commit for migrate in FLATPAKS_TO_MIGRATE: kind = migrate['kind'] old_origin = migrate['old-origin'] name = migrate.get('name') prefix = migrate.get('prefix') assert name or prefix old_branch = migrate.get('old-branch') # we build what should be the same list twice - from the flatpak # perspective by examining the installation and deployed apps, and from # the ostree perspective by examining the refs in the repo. ordinarily # we would only see them differ due to a bug in previous versions of # this script where the refs were not renamed during a migration. # everything in the matching_ostree_refs list will be copied to its # new name if we change (or previously changed) the branch or origin, # and then deleted after the deployed apps are updated. (T22763) matching_refs = _filter_refs(refs, name=name, prefix=prefix, kind=kind, branch=old_branch, origin=old_origin) matching_ostree_refs = _filter_ostree_refs(ostree_refs.keys(), old_origin, kind, name=name, prefix=prefix, branch=old_branch) # also search for name.* runtimes to migrate to catch extensions # unless we are already searching runtimes by prefix - in this case # we will already have found the matching extensions if name or kind == Flatpak.RefKind.APP: if not prefix: prefix = name + '.' matching_refs += _filter_refs(refs, prefix=prefix, kind=Flatpak.RefKind.RUNTIME, branch=old_branch, origin=old_origin) matching_ostree_refs += _filter_ostree_refs(ostree_refs.keys(), old_origin, Flatpak.RefKind.RUNTIME, prefix=prefix, branch=old_branch) if len(matching_refs) == 0: if name: logging.debug('Found no matches to migrate for {} {}' .format(kind.value_nick, name)) else: logging.debug('Found no matches to migrate for {} {}*' .format(kind.value_nick, prefix)) if 'new-name' in migrate: assert 'name' in migrate if len(matching_refs) > 1: logging.warning('Avoiding undefined behaviour, cannot rename ' 'an app with extensions: {}'.format(name)) continue for ref in matching_refs: origin = ref.get_origin() name = ref.get_name() arch = ref.get_arch() branch = ref.get_branch() old_ostree_ref = '{}:{}'.format(origin, ref.format_ref()) new_origin = migrate.get('new-origin', origin) new_name = migrate.get('new-name', name) new_branch = migrate.get('new-branch', branch) new_ostree_ref = '{}:{}/{}/{}/{}'.format(new_origin, kind.value_nick, new_name, arch, new_branch) mangle_metadata = migrate.get('mangle-metadata') logging.info('Found {} to migrate to {}'.format(old_ostree_ref, new_ostree_ref)) # if we're not changing anything, why are we here? assert (old_ostree_ref != new_ostree_ref) new_refs = _filter_refs(refs, name=new_name, kind=kind, arch=arch, branch=new_branch) if new_refs: new_ref = new_refs[0] else: new_ref = None try: vendor_prefixes = frozenset() migrate_extra = is_extra_data(repo, ostree_refs[old_ostree_ref]) # copy the old ref to the new one, pointing at the same commit if old_ostree_ref in ostree_refs and not new_ref: logging.info('Copying OSTree ref {} [{}] to {}' .format(old_ostree_ref, ostree_refs[old_ostree_ref], new_ostree_ref)) repo.prepare_transaction() try: kwargs = {} kwargs["mangle_metadata"] = mangle_metadata if kind == Flatpak.RefKind.APP: kwargs['old_app_id'] = name kwargs['new_app_id'] = new_name kwargs['migrate_extra'] = migrate_extra new_commit, vendor_prefixes = copy_commit( repo, ostree_refs[old_ostree_ref], new_ostree_ref.split(':')[1], **kwargs, ) repo.commit_transaction() finally: repo.abort_transaction() repo.set_ref_immediate(new_origin, new_ostree_ref.split(':')[1], new_commit) ostree_refs[new_ostree_ref] = new_commit if new_branch != branch or new_name != name: if new_ref: logging.info('Found already installed: {}' .format(new_ref.format_ref())) # adopt installed origin if app was already installed new_origin = new_ref.get_origin() else: logging.info( 'Deploying with new ref: {}'.format(new_ostree_ref) ) new_ref = inst.install_full(Flatpak.InstallFlags.NO_PULL | Flatpak.InstallFlags.NO_TRIGGERS, new_origin, kind, new_name, arch, new_branch) refs.append(new_ref) if migrate_extra: migrate_extra_data(ref, new_ref) logging.info('Uninstalling old ref: {}'.format(old_ostree_ref)) inst.uninstall_full(Flatpak.UninstallFlags.NO_PRUNE | Flatpak.UninstallFlags.NO_TRIGGERS, kind, name, arch, branch) triggers_needed = True # TODO: ideally we would defer the removal of the old app # until after the deploy file edits had been completed, so # we could pick up any prior previous-ids and carry them # over. however at present we are performing no such repeat # migrations (and may never need to). ref = new_ref origin = new_origin deploy_dir = ref.get_deploy_dir() deploy = os.path.join(deploy_dir, 'deploy') if new_origin != origin: update_deploy_file_with_origin(deploy, new_origin) # If, for app "org.example.Foo", we found a vendor-prefixed desktop file # like exports/share/applications/kde4/org.example.Foo.Bar.desktop, # pretend that "kde4-org.example.Foo" was an old ID of this application. # This will allow the X-Flatpak-RenamedFrom key to continue to include # the vendor-prefixed name across app updates. if new_name != name or vendor_prefixes: previous_ids = {name} for vendor_prefix in vendor_prefixes: previous_ids.add(vendor_prefix + '-' + name) update_deploy_file_with_previous_ids(deploy, previous_ids) except Exception: logging.exception('Failure applying migration to {}' .format(name)) # preserve the old ref so that the migration can be re-attempted if old_ostree_ref in matching_ostree_refs: matching_ostree_refs.remove(old_ostree_ref) # any refs we copied, along with any matches for apps we migrated # in the past, should be deleted. for old_ostree_ref in matching_ostree_refs: try: logging.info('Deleting old/stale OSTree ref {} [{}]' .format(old_ostree_ref, ostree_refs[old_ostree_ref])) repo.set_ref_immediate(old_ostree_ref.split(':')[0], old_ostree_ref.split(':')[1], None) except Exception: logging.exception('Failure deleting ref {}' .format(old_ostree_ref)) continue if triggers_needed: inst.run_triggers() if __name__ == '__main__': # Send logging messages both to the console and the journal handlers = [journal.JournalHandler()] if not GLib.log_writer_is_journald(sys.stderr.fileno()): handlers.append(logging.StreamHandler(sys.stderr)) logging.basicConfig(level=logging.INFO, handlers=handlers) parser = argparse.ArgumentParser() parser.add_argument('--debug', '-d', dest='debug', action='store_true') parsed_args, otherargs = parser.parse_known_args() if parsed_args.debug: logging.root.setLevel(logging.DEBUG) # ensure default remotes exist, such as eos-sdk and flathub _add_flatpak_repos() # remove any unused refs, remotes and runtimes _remove_ostree_refs() _remove_remotes() _remove_runtimes() # fix incorrect remote URLs _fix_remote_urls() # move apps and runtimes between branches and origins as specified above _migrate_installed_flatpaks()