Add historic package lists

content_resolver.py

@@ -89,9 +89,9 @@ def main():
 
     query = Query(data, configs, settings)
 
+    generate_historic_data(query)
     generate_pages(query)
     generate_data_files(query)
-    generate_historic_data(query)
 
 
     # -------------------------------------------------

content_resolver/historia_data.py

@@ -92,6 +92,133 @@ def _save_current_historic_data(query):
     log("  Done!")
     log("")
 
+def _save_current_historic_data_daily(query):
+    # Daily historic data for charts and package lists
+
+    log("Generating current daily historic data...")
+
+    # Where to save it
+    now = datetime.datetime.now()
+    day_of_year = now.strftime("%j")
+    year = now.strftime("%Y")
+    filename = "historic_data-{year}-day_{day}.json".format(
+        year=year,
+        day=day_of_year
+    )
+    output_dir = os.path.join(query.settings["output"], "history")
+    os.makedirs(output_dir, exist_ok=True)
+    file_path = os.path.join(output_dir, filename)
+
+    # What to save there
+    history_data = {
+        "date": str(now.strftime("%Y-%m-%d")),
+        "workloads": {},
+        "envs": {},
+        "repos": {},
+        "views": {},
+    }
+
+    # Workloads
+    for workload_id in query.workloads(None,None,None,None,list_all=True):
+        workload = query.data["workloads"][workload_id]
+
+        if not workload["succeeded"]:
+            continue
+
+        workload_history = {
+            "size": query.workload_size_id(workload_id),
+            "pkg_count": len(query.workload_pkgs_id(workload_id)),
+        }
+
+        history_data["workloads"][workload_id] = workload_history
+
+    # Environments
+    for env_id in query.envs(None,None,None,list_all=True):
+        env = query.data["envs"][env_id]
+
+        if not env["succeeded"]:
+            continue
+
+        env_history = {
+            "size": query.env_size_id(env_id),
+            "pkg_count": len(query.env_pkgs_id(env_id)),
+        }
+
+        history_data["envs"][env_id] = env_history
+
+    # Repositories
+    for repo_id in query.configs["repos"].keys():
+        history_data["repos"][repo_id] = {}
+
+        for arch, pkgs in query.data["pkgs"][repo_id].items():
+
+            repo_history = {
+                "pkg_count": len(pkgs),
+            }
+
+            history_data["repos"][repo_id][arch] = repo_history
+
+    # Views
+    for view_conf_id, view_conf in query.configs["views"].items():
+        view_all_arches = query.data["views_all_arches"][view_conf_id]
+
+        # Chart metrics
+        view_data = {
+            "srpm_count_env": view_all_arches["numbers"]["srpms"]["env"],
+            "srpm_count_req": view_all_arches["numbers"]["srpms"]["req"],
+            "srpm_count_dep": view_all_arches["numbers"]["srpms"]["dep"],
+        }
+
+        if view_all_arches["has_buildroot"]:
+            view_data["srpm_count_build_base"] = view_all_arches["numbers"]["srpms"]["build_base"]
+            view_data["srpm_count_build_level_1"] = view_all_arches["numbers"]["srpms"]["build_level_1"]
+            view_data["srpm_count_build_level_2_plus"] = view_all_arches["numbers"]["srpms"]["build_level_2_plus"]
+
+        # Package lists — sets remove duplicates across arches
+        all_source_names = set()
+        all_source_nevrs = set()
+        runtime_source_names = set()
+        runtime_source_nevrs = set()
+        buildroot_source_names = set()
+        buildroot_source_nevrs = set()
+
+        for arch in view_conf["architectures"]:
+            view_id = f"{view_conf_id}:{arch}"
+            view = query.data["views"][view_id]
+
+            for pkg_id, pkg in view["pkgs"].items():
+                source_name = pkg["source_name"]
+                source_nevr = pkg["sourcerpm"].rsplit(".src.rpm")[0]
+
+                all_source_names.add(source_name)
+                all_source_nevrs.add(source_nevr)
+
+                # Runtime if in a workload, buildroot-only otherwise
+                if pkg["in_workload_ids_all"]:
+                    runtime_source_names.add(source_name)
+                    runtime_source_nevrs.add(source_nevr)
+                else:
+                    buildroot_source_names.add(source_name)
+                    buildroot_source_nevrs.add(source_nevr)
+
+        # Sorted for consistent output
+        view_data["all_source_names"] = sorted(list(all_source_names))
+        view_data["all_source_nevrs"] = sorted(list(all_source_nevrs))
+        view_data["runtime_source_names"] = sorted(list(runtime_source_names))
+        view_data["runtime_source_nevrs"] = sorted(list(runtime_source_nevrs))
+        view_data["buildroot_source_names"] = sorted(list(buildroot_source_names))
+        view_data["buildroot_source_nevrs"] = sorted(list(buildroot_source_nevrs))
+
+        history_data["views"][view_conf_id] = view_data
+
+    # And save it
+    log("  Saving in: {file_path}".format(
+        file_path=file_path
+    ))
+    dump_data(file_path, history_data)
+
+    log("  Done!")
+    log("")
 
 def _read_historic_data(query):
     log("Reading historic data...")
@@ -134,8 +261,45 @@ def _read_historic_data(query):
     log("  Done!")
     log("")
 
+def _read_historic_data_daily(query):
+    # Read all daily snapshot files from output/history/
+
+    log("Reading daily historic data...")
+
+    directory = os.path.join(query.settings["output"], "history")
+
+    # Filter for daily files only (not weekly)
+    all_filenames = os.listdir(directory)
+    valid_filenames = []
+    for filename in all_filenames:
+        if bool(re.match(r"historic_data-....-day_....json", filename)):
+            valid_filenames.append(filename)
+    valid_filenames.sort()
+
+    # Get the data
+    historic_data_daily = {}
+
+    for filename in valid_filenames:
+        with open(os.path.join(directory, filename), "r") as file:
+            try:
+                document = json.load(file)
+                date = document["date"]
+            except (KeyError, ValueError):
+                err_log("Invalid file in daily historic data: {filename}. Ignoring.".format(
+                    filename=filename
+                ))
+                continue
+
+            historic_data_daily[date] = document
+            # Store filename so the browser knows which file to fetch for this date
+            historic_data_daily[date]["_filename"] = filename
+
+    log("  Done!")
+    log("")
+
+    return historic_data_daily
 
-def _generate_chartjs_data(historic_data, query):
+def _generate_chartjs_data(historic_data, query, prefix=""):
 
     # Data for workload pages
     for workload_id in query.workloads(None, None, None, None, list_all=True):
@@ -172,7 +336,8 @@ def _generate_chartjs_data(historic_data, query):
 
         entry_data["datasets"].append(dataset)
 
-        entry_name = "chartjs-data--workload--{workload_id}".format(
+        entry_name = "chartjs-data{prefix}--workload--{workload_id}".format(
+            prefix=prefix,
             workload_id=workload_id
         )
         _generate_json_file(entry_data, entry_name, query.settings)
@@ -219,7 +384,8 @@ def _generate_chartjs_data(historic_data, query):
 
                 entry_data["datasets"].append(dataset)
 
-            entry_name = "chartjs-data--workload-overview--{workload_conf_id}--{repo_id}".format(
+            entry_name = "chartjs-data{prefix}--workload-overview--{workload_conf_id}--{repo_id}".format(
+                prefix=prefix,
                 workload_conf_id=workload_conf_id,
                 repo_id=repo_id
             )
@@ -270,7 +436,8 @@ def _generate_chartjs_data(historic_data, query):
 
                     entry_data["datasets"].append(dataset)
 
-                entry_name = "chartjs-data--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format(
+                entry_name = "chartjs-data{prefix}--workload-cmp-arches--{workload_conf_id}--{env_conf_id}--{repo_id}".format(
+                    prefix=prefix,
                     workload_conf_id=workload_conf_id,
                     env_conf_id=env_conf_id,
                     repo_id=repo_id
@@ -322,7 +489,8 @@ def _generate_chartjs_data(historic_data, query):
 
                     entry_data["datasets"].append(dataset)
 
-                entry_name = "chartjs-data--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format(
+                entry_name = "chartjs-data{prefix}--workload-cmp-envs--{workload_conf_id}--{repo_id}--{arch}".format(
+                    prefix=prefix,
                     workload_conf_id=workload_conf_id,
                     repo_id=repo_id,
                     arch=arch
@@ -365,7 +533,8 @@ def _generate_chartjs_data(historic_data, query):
 
         entry_data["datasets"].append(dataset)
 
-        entry_name = "chartjs-data--env--{env_id}".format(
+        entry_name = "chartjs-data{prefix}--env--{env_id}".format(
+            prefix=prefix,
             env_id=env_id
         )
         _generate_json_file(entry_data, entry_name, query.settings)
@@ -412,7 +581,8 @@ def _generate_chartjs_data(historic_data, query):
 
                 entry_data["datasets"].append(dataset)
 
-            entry_name = "chartjs-data--env-overview--{env_conf_id}--{repo_id}".format(
+            entry_name = "chartjs-data{prefix}--env-overview--{env_conf_id}--{repo_id}".format(
+                prefix=prefix,
                 env_conf_id=env_conf_id,
                 repo_id=repo_id
             )
@@ -460,7 +630,8 @@ def _generate_chartjs_data(historic_data, query):
 
                 entry_data["datasets"].append(dataset)
 
-            entry_name = "chartjs-data--env-cmp-arches--{env_conf_id}--{repo_id}".format(
+            entry_name = "chartjs-data{prefix}--env-cmp-arches--{env_conf_id}--{repo_id}".format(
+                prefix=prefix,
                 env_conf_id=env_conf_id,
                 repo_id=repo_id
             )
@@ -552,12 +723,12 @@ def _generate_chartjs_data(historic_data, query):
 
             entry_data["datasets"].append(dataset)
 
-        entry_name = "chartjs-data--view--{view_conf_id}".format(
+        entry_name = "chartjs-data{prefix}--view--{view_conf_id}".format(
+            prefix=prefix,
             view_conf_id=view_conf_id
         )
         _generate_json_file(entry_data, entry_name, query.settings)
 
-
 def generate_historic_data(query):
     log("")
     log("###############################################################################")
@@ -566,13 +737,21 @@ def generate_historic_data(query):
     log("")
 
     # Step 1: Save current data
-    _save_current_historic_data(query)
+    _save_current_historic_data(query)  
+    _save_current_historic_data_daily(query)
 
     # Step 2: Read historic data
     historic_data = _read_historic_data(query)
+    historic_data_daily = _read_historic_data_daily(query)
 
     # Step 3: Generate Chart.js data
     _generate_chartjs_data(historic_data, query)
+    _generate_chartjs_data(historic_data_daily, query, prefix="-daily")
+
+    # Date-to-filename map for the browser to fetch daily snapshots
+    query.data["historic_daily_dates"] = {
+        date: entry["_filename"] for date, entry in historic_data_daily.items()
+    }
 
     log("Done!")
     log("")

content_resolver/page_generation.py

@@ -400,7 +400,9 @@ def _generate_view_pages(query):
         template_data = {
             "query": query,
             "view_conf": view_conf,
-            "view_all_arches": view_all_arches
+            "view_all_arches": view_all_arches,
+            # Date-to-filename map for the daily snapshots date picker
+            "historic_daily_dates": query.data.get("historic_daily_dates", {}),
         }
 
         # Generate the overview page

scripts/check_cdn_updates.py

@@ -457,6 +457,15 @@ def parse_layout_html(file_path: str) -> Tuple[Dict[str, str], Dict[str, List[st
                     urls["axios"] = []
                 urls["axios"].append(url)
 
+        # Flatpickr
+        elif "flatpickr" in url_lower:
+            version = extract_version_from_url(url, "flatpickr")
+            if version:
+                if "flatpickr" not in versions:
+                    versions["flatpickr"] = version
+                    urls["flatpickr"] = []
+                urls["flatpickr"].append(url)
+
     return versions, urls
 
 

templates/layout.html

@@ -27,6 +27,15 @@
     crossorigin="anonymous"></script>
   <!-- Chart stuff end -->
 
+  <!-- Calendar datepicker start -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/flatpickr@4.6.13/dist/flatpickr.min.css" 
+    integrity="sha384-RkASv+6KfBMW9eknReJIJ6b3UnjKOKC5bOUaNgIY778NFbQ8MtWq9Lr/khUgqtTt" 
+    crossorigin="anonymous">
+  <script src="https://cdn.jsdelivr.net/npm/flatpickr@4.6.13/dist/flatpickr.min.js" 
+    integrity="sha384-5JqMv4L/Xa0hfvtF06qboNdhvuYXUku9ZrhZh3bSk8VXF0A/RuSLHpLsSV9Zqhl6" 
+    crossorigin="anonymous"></script>
+  <!-- Calendar datepicker end -->
+
   <script src="https://cdn.jsdelivr.net/npm/axios@1.18.1/dist/axios.min.js"
     integrity="sha384-7068mr5076h07c4xiJhMQYkkKke2zXmg44bCAB5ZSVNaM+eqoepYOy9/mRKdYulG"
     crossorigin="anonymous"></script>