New encrypted wallet file header:

electrum/__init__.py

@@ -17,7 +17,7 @@ class GuiImportError(ImportError):
 from .version import ELECTRUM_VERSION
 from .util import format_satoshis
 from .wallet import Wallet
-from .storage import WalletStorage
+from .stored_dict import DictStorage
 from .coinchooser import COIN_CHOOSERS
 from .network import Network, pick_random_server
 from .interface import Interface

electrum/commands.py

@@ -360,7 +360,6 @@ async def password(self, password=None, new_password=None, encrypt_file=None, wa
             else:
                 encrypt_file = wallet.storage.is_encrypted()
         wallet.update_password(password, new_password, encrypt_storage=encrypt_file)
-        wallet.save_db()
         return {'password': wallet.has_password()}
 
     @command('w')
@@ -1553,7 +1552,6 @@ async def addtransaction(self, tx, wallet: Abstract_Wallet = None):
         tx = Transaction(tx)
         if not wallet.adb.add_transaction(tx):
             return False
-        wallet.save_db()
         return tx.txid()
 
     @command('w')
@@ -1667,7 +1665,6 @@ async def removelocaltx(self, txid, wallet: Abstract_Wallet = None):
                 f'Only local transactions can be removed. '
                 f'This tx has height: {height} != {TX_HEIGHT_LOCAL}')
         wallet.adb.remove_transaction(txid)
-        wallet.save_db()
 
     @command('wn')
     async def get_tx_status(self, txid, wallet: Abstract_Wallet = None):

electrum/crypto.py

@@ -134,9 +134,10 @@ def strip_PKCS7_padding(data: bytes) -> bytes:
     return data[0:-padlen]
 
 
-def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
+def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes, append_pkcs7=True) -> bytes:
     assert_bytes(key, iv, data)
-    data = append_PKCS7_padding(data)
+    if append_pkcs7:
+        data = append_PKCS7_padding(data)
     if HAS_CRYPTODOME:
         e = CD_AES.new(key, CD_AES.MODE_CBC, iv).encrypt(data)
     elif HAS_CRYPTOGRAPHY:
@@ -152,7 +153,7 @@ def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     return e
 
 
-def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
+def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes, strip_pkcs7=True) -> bytes:
     assert_bytes(key, iv, data)
     if HAS_CRYPTODOME:
         cipher = CD_AES.new(key, CD_AES.MODE_CBC, iv)
@@ -168,9 +169,11 @@ def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     else:
         raise Exception("no AES backend found")
     try:
-        return strip_PKCS7_padding(data)
+        if strip_pkcs7:
+            data = strip_PKCS7_padding(data)
     except InvalidPadding:
         raise InvalidPassword()
+    return data
 
 
 def EncodeAES_bytes(secret: bytes, msg: bytes) -> bytes:

electrum/daemon.py

@@ -47,7 +47,7 @@
     log_exceptions, randrange, OldTaskGroup, UserFacingException, JsonRPCError, os_chmod
 )
 from .wallet import Wallet, Abstract_Wallet
-from .storage import WalletStorage
+from .stored_dict import DictStorage
 from .wallet_db import WalletDB, WalletUnfinished
 from .commands import known_commands, Commands
 from .simple_config import SimpleConfig
@@ -544,15 +544,14 @@ def _load_wallet(
             force_check_password: bool = False,  # if set, always validate password
     ) -> Optional[Abstract_Wallet]:
         path = standardize_path(path)
-        storage = WalletStorage(path, allow_partial_writes=config.WALLET_PARTIAL_WRITES)
-        if not storage.file_exists():
+        if not os.path.exists(path):
             raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+        storage = DictStorage(path, allow_partial_writes=config.WALLET_PARTIAL_WRITES)
         if storage.is_encrypted():
             if not password:
                 raise InvalidPassword('No password given')
             storage.decrypt(password)
-        # read data, pass it to db
-        db = WalletDB(storage.read(), storage=storage, upgrade=upgrade)
+        db = WalletDB(storage, upgrade=upgrade)
         if db.get_action():
             raise WalletUnfinished(db)
         wallet = Wallet(db, config=config)

electrum/gui/qml/qedaemon.py

@@ -13,7 +13,7 @@
 from electrum.lnchannel import ChannelState
 from electrum.bitcoin import is_address
 from electrum.bitcoin import verify_usermessage_with_address
-from electrum.storage import StorageReadWriteError, WalletStorage
+from electrum.stored_dict import StorageReadWriteError, DictStorage
 
 from .auth import AuthMixin, auth_protect
 from .qefx import QEFX
@@ -333,7 +333,7 @@ def isValidWalletName(self, wallet_name: str) -> bool:
         wallet_path = self.wallet_path_from_wallet_name(wallet_name)
         # validate that the path looks sane to the filesystem:
         try:
-            temp_storage = WalletStorage(wallet_path)
+            temp_storage = DictStorage(wallet_path, init_db=False)
         except (StorageReadWriteError, WalletFileException):
             return False
         except Exception:

electrum/gui/qml/qetxdetails.py

@@ -498,7 +498,6 @@ def removeLocalTx(self, confirm=False):
             return
 
         self._wallet.wallet.adb.remove_transaction(txid)
-        self._wallet.wallet.save_db()
 
         # NOTE: from here, the tx/txid is unknown and all properties are invalid.
         # UI should close TxDetails and avoid interacting with this qetxdetails instance.

electrum/gui/qml/qewallet.py

@@ -637,7 +637,6 @@ def save_tx(self, tx: 'PartialTransaction') -> bool:
                 self.saveTxError.emit(tx.txid(), 'conflict',
                             _("Transaction could not be saved.") + "\n" + _("It conflicts with current history."))
                 return False
-            self.wallet.save_db()
             self.saveTxSuccess.emit(tx.txid())
             self.historyModel.initModel(True)
             return True
@@ -754,7 +753,8 @@ def setPassword(self, password):
 
         try:
             self._logger.info('setting new password')
-            self.wallet.update_password(current_password, password, encrypt_storage=True)
+            encrypt_storage = self.wallet.storage.supports_file_encryption()
+            self.wallet.update_password(current_password, password, encrypt_storage=encrypt_storage)
             # restore the invariant that all loaded wallets in qml must be unlocked:
             self.wallet.unlock(password)
             return True

electrum/gui/qt/__init__.py

@@ -507,7 +507,7 @@ def _start_wizard_to_select_or_create_wallet(self, path) -> Optional[Abstract_Wa
                 self.logger.info('wizard dialog cancelled by user')
                 return
             db.put('x3', wizard.get_wizard_data()['x3'])
-            db.write_and_force_consolidation()  # TODO API for db is a bit weird: there should be a close method
+            db.storage.write()  # TODO API for db is a bit weird: there should be a close method
 
         wallet = self.daemon.load_wallet(wallet_file, password, upgrade=True)
         return wallet

electrum/gui/qt/history_list.py

@@ -810,7 +810,6 @@ def remove_local_tx(self, tx_hash: str):
         if not self.main_window.question(msg=question, title=_("Please confirm")):
             return
         self.wallet.adb.remove_transaction(tx_hash)
-        self.wallet.save_db()
         # need to update at least: history_list, utxo_list, address_list
         self.main_window.need_update.set()
 

electrum/gui/qt/invoice_list.py

@@ -213,6 +213,5 @@ def show_log(self, key, log: Sequence[HtlcLog]):
 
     def delete_invoices(self, keys):
         for key in keys:
-            self.wallet.delete_invoice(key, write_to_disk=False)
+            self.wallet.delete_invoice(key)
             self.delete_item(key)
-        self.wallet.save_db()

electrum/gui/qt/main_window.py

@@ -51,6 +51,7 @@
 from electrum.gui import messages
 from electrum import (keystore, constants, util, bitcoin, commands,
                       lnutil)
+from electrum.stored_dict import PasswordType
 from electrum.bitcoin import COIN, is_address, DummyAddress
 from electrum.plugin import run_hook
 from electrum.i18n import _
@@ -698,6 +699,26 @@ def select_backup_dir(self, b):
             self.config.WALLET_BACKUP_DIRECTORY = dirname
             self.backup_dir_e.setText(dirname)
 
+    def get_storage_password(self):
+        if self.wallet.has_storage_encryption():
+            if self.wallet.storage.is_encrypted_with_hw_device():
+                password = self.wallet.keystore.get_password_for_storage_encryption()
+                password_type = PasswordType.XPUB
+            else:
+                password_type = PasswordType.USER
+                while True:
+                    password = self.password_dialog(parent=self, msg='')
+                    if not password:
+                        raise UserCancelled
+                    try:
+                        self.wallet.storage.check_password(password)
+                    except InvalidPassword:
+                        continue
+                    break
+        else:
+            password, password_type = None, None
+        return password, password_type
+
     def backup_wallet(self):
         d = WindowModalDialog(self, _("File Backup"))
         vbox = QVBoxLayout(d)
@@ -725,8 +746,16 @@ def backup_wallet(self):
         if backup_dir is None:
             self.show_message(_("You need to configure a backup directory in your preferences"), title=_("Backup not configured"))
             return
+        new_path = os.path.join(backup_dir, self.wallet.basename() + '.backup')
+        if os.path.exists(new_path):
+            self.show_message(f'File already exists: {new_path}')
+            return
+        try:
+            password, password_type = self.get_storage_password()
+        except UserCancelled:
+            return
         try:
-            new_path = self.wallet.save_backup(backup_dir)
+            self.wallet.save_backup(new_path, password, password_type)
         except BaseException as reason:
             self.show_critical(_("Electrum was unable to copy your wallet file to the specified location.") + "\n" + str(reason), title=_("Unable to create backup"))
             return
@@ -1748,7 +1777,7 @@ def save_notes_text(self):
 
     def update_console(self):
         console = self.console
-        console.history = self.wallet.db.get_stored_item("qt-console-history", [])
+        console.history = self.wallet.db.get_list("qt-console-history")
         console.history_index = len(console.history)
 
         console.updateNamespace({
@@ -1922,14 +1951,13 @@ def update_buttons_on_seed(self):
         self.password_button.setVisible(self.wallet.may_have_password())
 
     def change_password_dialog(self):
-        from electrum.stored_dict import StorageEncryptionVersion
-        if StorageEncryptionVersion.XPUB_PASSWORD in self.wallet.get_available_storage_encryption_versions():
+        if self.wallet.is_hw_encryption_available():
             from .password_dialog import ChangePasswordDialogForHW
             d = ChangePasswordDialogForHW(self, self.wallet)
             ok, old_password, new_password, encrypt_with_xpub = d.run()
             if not ok:
                 return
-            has_xpub_encryption = self.wallet.storage.get_encryption_version() == StorageEncryptionVersion.XPUB_PASSWORD
+            has_xpub_encryption = self.wallet.storage.is_encrypted_with_hw_device()
             def on_password(hw_dev_pw):
                 self._update_wallet_password(
                     old_password = hw_dev_pw if has_xpub_encryption else old_password,
@@ -1950,8 +1978,12 @@ def on_password(hw_dev_pw):
         self.update_lock_menu()
 
     def _update_wallet_password(self, *, old_password, new_password, xpub_encrypt=False):
+        encrypt_storage = self.wallet.storage.supports_file_encryption()
         try:
-            self.wallet.update_password(old_password, new_password, encrypt_storage=True, xpub_encrypt=xpub_encrypt)
+            self.wallet.update_password(
+                old_password, new_password,
+                encrypt_storage=encrypt_storage,
+                xpub_encrypt=xpub_encrypt)
         except InvalidPassword as e:
             self.show_error(str(e))
             return
@@ -2947,7 +2979,6 @@ def save_transaction_into_wallet(self, tx: Transaction):
             win.show_error(e)
             return False
         else:
-            self.wallet.save_db()
             # need to update at least: history_list, utxo_list, address_list
             self.need_update.set()
             msg = (_("Transaction added to wallet history.") + '\n\n' +

electrum/gui/qt/wizard/wallet.py

@@ -15,13 +15,13 @@
 from electrum.i18n import _
 from electrum.keystore import bip44_derivation, bip39_to_seed, purpose48_derivation, ScriptTypeNotSupported
 from electrum.plugin import run_hook, HardwarePluginLibraryUnavailable
-from electrum.storage import StorageReadWriteError
+from electrum.storage import StorageReadWriteError, StorageException
 from electrum.util import WalletFileException, get_new_wallet_name, UserFacingException, InvalidPassword
 from electrum.util import is_subpath, ChoiceItem, multisig_type, UserCancelled, standardize_path
 from electrum.wallet import wallet_types
 from .wizard import QEAbstractWizard, WizardComponent
 from electrum.logging import get_logger, Logger
-from electrum import WalletStorage, mnemonic, keystore
+from electrum import DictStorage, mnemonic, keystore
 from electrum.wallet_db import WalletDB
 from electrum.wizard import NewWalletWizard, KeystoreWizard, WizardViewState
 
@@ -176,7 +176,7 @@ def is_finalized(self, wizard_data: dict) -> bool:
 
         wallet_file = wizard_data['wallet_name']
 
-        storage = WalletStorage(wallet_file)
+        storage = DictStorage(wallet_file)
         assert storage.file_exists(), f"file {wallet_file!r} does not exist"
         if not storage.is_encrypted_with_user_pw() and not storage.is_encrypted_with_hw_device():
             return True
@@ -280,7 +280,7 @@ def __init__(self, parent, wizard):
         self.layout().addLayout(hbox2)
         self.layout().addStretch(1)
 
-        temp_storage = None  # type: Optional[WalletStorage]
+        temp_storage = None  # type: Optional[DictStorage]
         datadir_wallet_folder = self.wizard.config.get_datadir_wallet_path()
 
         def relative_path(path):
@@ -313,12 +313,12 @@ def on_filename(filename_or_path):
                 wallet_from_memory = self.wizard._daemon.get_wallet(_path)
                 try:
                     if wallet_from_memory:
-                        temp_storage = wallet_from_memory.storage  # type: Optional[WalletStorage]
+                        temp_storage = wallet_from_memory.storage  # type: Optional[DictStorage]
                         self.wallet_is_open = True
                     else:
-                        temp_storage = WalletStorage(_path)
+                        temp_storage = DictStorage(_path, init_db=False)
                     self.wallet_exists = temp_storage.file_exists()
-                except (StorageReadWriteError, WalletFileException) as e:
+                except (StorageReadWriteError, StorageException) as e:
                     msg = _('Cannot read file') + f'\n{repr(e)}'
                 except Exception as e:
                     self.logger.exception('')
@@ -327,7 +327,7 @@ def on_filename(filename_or_path):
                 msg = ""
             self.valid = temp_storage is not None
             user_needs_to_enter_password = False
-            if temp_storage:
+            if temp_storage is not None:
                 if not temp_storage.file_exists():
                     msg = _("This file does not exist.") + '\n' \
                           + _("Press 'Next' to create this wallet, or choose another file.")
@@ -1376,7 +1376,7 @@ def validate(self):
     def check_hw_decrypt(self):
         wallet_file = self.wizard_data['wallet_name']
 
-        storage = WalletStorage(wallet_file)
+        storage = DictStorage(wallet_file)
         if not storage.is_encrypted_with_hw_device():
             return True
 

electrum/gui/stdio.py

@@ -6,7 +6,7 @@
 
 from electrum.gui import BaseElectrumGui
 from electrum import util
-from electrum import WalletStorage, Wallet
+from electrum import DictStorage, Wallet
 from electrum.wallet import Abstract_Wallet
 from electrum.wallet_db import WalletDB
 from electrum.util import format_satoshis, EventListener, event_listener
@@ -26,7 +26,7 @@ class ElectrumGui(BaseElectrumGui, EventListener):
     def __init__(self, *, config, daemon, plugins):
         BaseElectrumGui.__init__(self, config=config, daemon=daemon, plugins=plugins)
         self.network = daemon.network
-        storage = WalletStorage(config.get_wallet_path())
+        storage = DictStorage(config.get_wallet_path())
         password = None
         if not storage.file_exists():
             print("Wallet not found. try 'electrum create'")

electrum/gui/text.py

@@ -21,7 +21,7 @@
 from electrum.transaction import PartialTxOutput
 from electrum.wallet import Wallet, Abstract_Wallet
 from electrum.wallet_db import WalletDB
-from electrum.storage import WalletStorage
+from electrum.stored_dict import DictStorage
 from electrum.network import NetworkParameters, TxBroadcastError, BestEffortRequestFailed, ProxySettings
 from electrum.interface import ServerAddr
 from electrum.invoices import Invoice
@@ -62,7 +62,7 @@ class ElectrumGui(BaseElectrumGui, EventListener):
     def __init__(self, *, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'):
         BaseElectrumGui.__init__(self, config=config, daemon=daemon, plugins=plugins)
         self.network = daemon.network
-        storage = WalletStorage(config.get_wallet_path())
+        storage = DictStorage(config.get_wallet_path())
         password = None
         if not storage.file_exists():
             print("Wallet not found. try 'electrum create'")

electrum/invoices.py

@@ -236,7 +236,7 @@ def get_id(self) -> str:
         else:  # on-chain
             return get_id_from_onchain_outputs(outputs=self.get_outputs(), timestamp=self.time)
 
-    def as_dict(self, status):
+    def export(self, status):
         d = {
             'is_lightning': self.is_lightning(),
             'amount_BTC': format_satoshis(self.get_amount_sat()),
@@ -298,7 +298,7 @@ def can_be_paid_onchain(self) -> bool:
             return True
 
     def to_debug_json(self) -> Dict[str, Any]:
-        d = self.to_json()
+        d = self.as_dict()
         d["lnaddr"] = self._lnaddr.to_debug_json()
         return d