Skip to content

LE renewal: on config file errors reissue the cert #3324 #3325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkf-sysangels wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

LE renewal: on config file errors reissue the cert #3324 #3325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 109 additions & 34 deletions src/common/core/letsencrypt/jobs/certbot-new.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@
build_certbot_env,
get_expected_acme_directory,
prepare_logs_dir,
remove_empty_renewal_configs,
repair_live_symlinks,
resolve_certbot_entrypoint,
)

Expand All @@ -93,6 +95,11 @@
RUNNING_CERTBOT = 0
MONITOR_STOP = Event()
MONITOR_THREAD: Optional[Thread] = None
# NOTE: certbot-new is executed as a script in normal operation, but certbot-renew
# imports it (via importlib) to reuse issuance logic when it needs to reissue after
# repairing a broken renewal state. To keep imports side-effect free, script execution
# is gated under __main__ and helpers accept an optional job instance.
JOB = None # Set by main(); reissue_for_removed_configs passes job explicitly when called from certbot-renew


def _monitor_certbot_progress() -> None:
Expand Down Expand Up @@ -631,6 +638,61 @@ def _determine_wildcard_bases(labels_list: List[List[str]]) -> Set[str]:
return bases


def load_services() -> Dict[str, Dict[str, Union[str, bool, int, Dict[str, str]]]]:
"""Build the services dict from SERVER_NAME and per-service env. Used by main() and reissue_for_removed_configs()."""
server_names = getenv("SERVER_NAME", "www.example.com").strip()
if not server_names:
return {}
services: Dict[str, Dict[str, Union[str, bool, int, Dict[str, str]]]] = {}
for service in server_names.split():
if not service.strip():
continue
for cert_name, config in build_service_entries(service).items():
if cert_name in services:
if certificate_fingerprint(services[cert_name]) == certificate_fingerprint(config):
merged = normalize_server_names(services[cert_name]["server_names"]) | normalize_server_names(config["server_names"])
services[cert_name]["server_names"] = format_server_names(merged)
continue
LOGGER.warning(f"[Service: {cert_name}] Certificate configuration conflict detected, keeping first occurrence.")
continue
services[cert_name] = config
return services


def reissue_for_removed_configs(
removed_cert_names: List[str],
job,
cmd_env: Dict[str, str],
logger,
) -> bool:
"""
Reissue certificates for services whose renewal config was removed (empty or broken).
Uses service definitions from env (load_services), not the failing config file.
Called from certbot-renew when certbot-new runs only once at init.
"""
services = load_services()
if not services:
return True
to_reissue: List[Tuple[str, Dict[str, Union[str, bool, int, Dict[str, str]]]]] = []
for cert_name in removed_cert_names:
# Renewal filenames can include certbot profile suffixes (-ecdsa/-rsa). We resolve both
# the stripped and unstripped variants to find the correct service key to reissue.
service_name = (
cert_name[:-6] if cert_name.endswith("-ecdsa") else (cert_name[:-4] if cert_name.endswith("-rsa") else cert_name)
)
key = service_name if service_name in services else (cert_name if cert_name in services else None)
if key is not None and services[key].get("activated"):
to_reissue.append((key, services[key]))
if not to_reissue:
return True
for service_name, config in to_reissue:
config["force_renew"] = True
logger.info(f"Reissuing certificate for {service_name} (broken renewal config was removed, using service definition).")
if not generate_certificate(service_name, config, cmd_env, job=job):
return False
return True


def certbot_delete(service: str, cmd_env: Dict[str, str] = None) -> int:
# * Building the certbot command
command = [
Expand Down Expand Up @@ -671,6 +733,7 @@ def certbot_new(
config: Dict[str, Union[str, bool, int, Dict[str, str]]],
cmd_env: Dict[str, str],
paths: CertbotPaths,
job=None,
) -> int:
certbot_entrypoint = resolve_certbot_entrypoint(
str(config.get("acme_server") or "letsencrypt"),
Expand Down Expand Up @@ -745,7 +808,7 @@ def certbot_new(
credentials_file = f"credentials_{bytes_hash(credentials, algorithm='sha256')[:12]}.{config['provider'].get_file_type()}"
credentials_path = CACHE_PATH.joinpath(credentials_file)
if not credentials_path.is_file() or config["force_renew"]:
JOB.cache_file(credentials_file, credentials)
(job or JOB).cache_file(credentials_file, credentials)
credentials_path.chmod(0o600) # Set permissions to read/write for owner only

# * Adding the credentials to the command
Expand Down Expand Up @@ -793,7 +856,12 @@ def certbot_new(
return process.returncode


def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, Dict[str, str]]], cmd_env: Dict[str, str]) -> bool:
def generate_certificate(
service: str,
config: Dict[str, Union[str, bool, int, Dict[str, str]]],
cmd_env: Dict[str, str],
job=None,
) -> bool:
LOGGER.info(
f"Asking{' wildcard' if config['wildcard'] else ''} certificates for domain(s) : {config['server_names']} (email = {config['email'] or 'not provided'}){' using staging' if config['staging'] else ''} with {config['challenge']} challenge, using {config['profile']!r} profile on {config['acme_server']}..."
)
Expand All @@ -819,7 +887,7 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D
global RUNNING_CERTBOT
RUNNING_CERTBOT += 1
try:
ret = certbot_new(service, config, cmd_env.copy(), paths)
ret = certbot_new(service, config, cmd_env.copy(), paths, job=job)
finally:
with RUNNING_LOCK:
RUNNING_CERTBOT -= 1
Expand All @@ -835,37 +903,28 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D
return False


try:
# ? Load services configuration
server_names = getenv("SERVER_NAME", "www.example.com").strip()

if not server_names:
def main() -> int:
status = 0
services = load_services()
if not services:
LOGGER.warning("No services found, exiting...")
sys_exit(0)

services = {}
for service in server_names.split():
if not service.strip():
continue

for cert_name, config in build_service_entries(service).items():
if cert_name in services:
if certificate_fingerprint(services[cert_name]) == certificate_fingerprint(config):
merged = normalize_server_names(services[cert_name]["server_names"]) | normalize_server_names(config["server_names"])
services[cert_name]["server_names"] = format_server_names(merged)
continue
LOGGER.warning(f"[Service: {cert_name}] Certificate configuration conflict detected, keeping first occurrence.")
continue
services[cert_name] = config

return 0
if not any(service["activated"] for service in services.values()):
LOGGER.info("No services uses Let's Encrypt, skipping generation of new certificates...")
sys_exit(0)
return 0

prepare_logs_dir(LOGS_DIR, LOGGER)

global JOB
JOB = Job(LOGGER, __file__.replace("new", "renew"))

# Repair live/ symlinks first, then remove broken renewal configs (empty, unparseable, or missing required file reference).
# Reissue uses service definitions from env (load_services), not the failing config file.
removed_empty_configs: List[str] = []
if DATA_PATH.is_dir():
repair_live_symlinks(DATA_PATH, LOGGER)
removed_empty_configs = remove_empty_renewal_configs(DATA_PATH, LOGGER)

# ? Fetch existing certificates
cmd_env = build_certbot_env(JOB, DEPS_PATH)

Expand Down Expand Up @@ -948,6 +1007,16 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D
zerossl_api_key_hashes = load_zerossl_api_key_hashes()
LOGGER_CERTBOT.debug(f"Stored ZeroSSL API key hashes: {list(zerossl_api_key_hashes.keys())}")

# If we removed broken renewal configs, force reissue for the affected services from service definition (cert_name can have -ecdsa/-rsa suffix)
for cert_name in removed_empty_configs:
service_name = (
cert_name[:-6] if cert_name.endswith("-ecdsa") else (cert_name[:-4] if cert_name.endswith("-rsa") else cert_name)
)
key = service_name if service_name in services else (cert_name if cert_name in services else None)
if key is not None and services[key].get("activated"):
services[key]["force_renew"] = True
LOGGER.info(f"Service {key}: broken renewal config was removed, will reissue certificate from service definition.")

# ? Check if the services' certificates already exist
for server_name, config in services.items():
if config["force_renew"] or not config["activated"] or server_name not in existing_certificates:
Expand Down Expand Up @@ -1107,11 +1176,17 @@ def generate_certificate(service: str, config: Dict[str, Union[str, bool, int, D
LOGGER.error(f"Error while saving data to db cache : {err}")
else:
LOGGER.info("Successfully saved data to db cache")
except SystemExit as e:
status = e.code
except BaseException as e:
status = 2
LOGGER.debug(format_exc())
LOGGER.error(f"Exception while running certbot-new.py :\n{e}")

sys_exit(status)
return status


if __name__ == "__main__":
status = 0
try:
status = main()
except SystemExit as e:
status = e.code if e.code is not None else 0
except BaseException as e:
status = 2
LOGGER.debug(format_exc())
LOGGER.error(f"Exception while running certbot-new.py :\n{e}")
sys_exit(status)
24 changes: 23 additions & 1 deletion src/common/core/letsencrypt/jobs/certbot-renew.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#!/usr/bin/env python3

import importlib.util
from os import getenv, sep
from os.path import join
from os.path import dirname, join
from pathlib import Path
from subprocess import DEVNULL, PIPE, Popen
from sys import exit as sys_exit, path as sys_path
from traceback import format_exc
Expand All @@ -22,6 +24,8 @@
build_certbot_env,
is_zerossl_used_in_env,
prepare_logs_dir,
remove_empty_renewal_configs,
repair_live_symlinks,
resolve_certbot_entrypoint,
)

Expand Down Expand Up @@ -50,7 +54,25 @@

JOB = Job(LOGGER, __file__)

# Repair live/ symlinks first (avoids certbot "expected to be a symlink"), then remove broken renewal configs.
# Broken = empty, unparseable, or missing required file reference. Reissue uses service definitions from env.
removed_empty_configs: list = []
if DATA_PATH.is_dir():
repair_live_symlinks(DATA_PATH, LOGGER)
removed_empty_configs = remove_empty_renewal_configs(DATA_PATH, LOGGER)

cmd_env = build_certbot_env(JOB, DEPS_PATH)

# certbot-new is scheduled as "every: once" (init-only), while certbot-renew is periodic.
# If we removed a broken renewal config, trigger reissue now using service definitions from env (not the bad file).
if removed_empty_configs:
certbot_new_path = Path(dirname(__file__)).resolve() / "certbot-new.py"
spec = importlib.util.spec_from_file_location("certbot_new", certbot_new_path)
certbot_new_mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(certbot_new_mod)
if not certbot_new_mod.reissue_for_removed_configs(removed_empty_configs, JOB, cmd_env, LOGGER):
status = 2
LOGGER.error("Reissue after removing broken renewal config(s) failed")
acme_server = "zerossl" if is_zerossl_used_in_env() else "letsencrypt"
certbot_entrypoint = resolve_certbot_entrypoint(
acme_server,
Expand Down
Loading