make script faster

app.py

@@ -10,8 +10,10 @@ import frontmatter
 from slugify import slugify
 import pypandoc
 
-# TODO make newsletter URL's absolute to klank.school
+import xml.etree.ElementTree as ET
 
+
+# TODO make newsletter URL's absolute to klank.school
 env = Environment(
     loader=PackageLoader("src"),
     autoescape=select_autoescape()
@@ -19,17 +21,12 @@ env = Environment(
 
 CONTENT_D = os.path.abspath("src/content")
 OUTPUT_D = "dist"
+OUT_ASSETS = "dist/assets"
+SRC_ASSETS = "src/assets"
 documents = {}
+now = datetime.now()
 
-def listDocuments(params):
+# Utils
-    param = params.split(" ")
-
-    template = env.select_template([f"snippets/list-documents.jinja"])
-    html = template.render(documents=documents, layout=param[0], type=param[1])
-
-    return html
-
-
 def getParam(params, index):
     if len(params) > index:
         return params[index]
@@ -37,9 +34,18 @@ def getParam(params, index):
         return False
 
 
+# jinja filter that can list documents
+def listDocuments(params):
+    param = params.split(" ")
+    template = env.select_template(["snippets/list-documents.jinja"])
+    html = template.render(documents=documents, layout=param[0], type=param[1])
+
+    return html
+
+# jinja filter that can list events
 def listEvents(params):
     param = params.split(" ")
-    tag=getParam(param, 1)
+    tag = getParam(param, 1)
 
     if "events" not in documents:
         return ""
@@ -52,20 +58,21 @@ def listEvents(params):
     else:
         events = documents["events"]
 
-    template = env.select_template([f"snippets/list-events.jinja"])
+    template = env.select_template(["snippets/list-events.jinja"])
     html = template.render(events=events, filter=param[0], tag=getParam(param, 1))
 
     return html
 
-
+# jinja filter to make a slug out of a stirng
 def slugify_filter(value):
     return slugify(value)
 
-
+# jinja filter for date formatting
 def prettydate(value, format='%d/%m/%Y'):
     return datetime.fromtimestamp(int(value)).strftime(format)
 
 
+# jinja filter to replace shortcodes in HTML
 def shortcode_filter(value):
 
     shortcode_callbacks = {
@@ -91,12 +98,9 @@ env.filters["shortcode"] = shortcode_filter
 env.filters["slugify"] = slugify_filter
 env.filters["prettydate"] = prettydate
 
-
+# translate a single file into HTML
-def render_single_file(template, page, path, dist, name = False):
+def render_single_file(page, path, dist, name = False):
-    modifier = False
+    template = env.select_template([f"{name}.jinja", "post.jinja"])
-
-    print(f"rendering: {path}")
-
     html = template.render(documents=documents, page=page, name=name)
     name = Path(path).stem
 
@@ -107,30 +111,65 @@ def render_single_file(template, page, path, dist, name = False):
         output_file.write(html)
 
 
+# find a pre-rendered page
+def get_existing_page(path, slug):
+    stem = Path(path).stem;
+    folder =  os.path.basename(os.path.dirname(path))
+
+    if stem == "index" and folder != "content":
+        folder = Path(path).parent.parent.name
+
+    if slug in documents:
+        return documents[slug]
+    
+    if folder == "content":
+        return False
+    
+    for doc in documents[folder]:
+        if doc["slug"] == slug:
+            return doc
+    
+    return False
+
+# build a slug including the folder
+def get_slug(path, folder, filename):
+    if folder == "content":
+        return slugify(filename)
+    else:
+        return slugify(f"{folder}/{filename}")
+
+# compile markdown into cited HTML
 def get_page_data(path, isPreload=False):
+
     filename = Path(path).stem
+    folder  = os.path.basename(os.path.dirname(path))
+    slug = get_slug(path, folder, filename)
+    
+    prerendered = get_existing_page(path, slug)
+
+    if prerendered:
+        return prerendered
 
     page = frontmatter.load(path)
-    page['slug'] = slugify(filename)
+    page['slug'] = slug
     page.filename = filename
-    page.folder = os.path.basename(os.path.dirname(path))
+    page.folder = folder
+    latex = page.content
 
     if "start_datetime" in page:
-        now = datetime.now()
         page["has_passed"] = datetime.fromtimestamp(page["start_datetime"]) < now
-        
 
-
+    
-
+    if ".include" in page.content:
-    latex = pypandoc.convert_text(
+        latex = pypandoc.convert_text(
-        page.content,
+            page.content,
-        to='md',
+            to='md',
-        format='md',
+            format='md',
-        extra_args=[
+            extra_args=[
-            "-N",
+                "-N",
-            "--section-divs",
+                "--section-divs",
-            "--lua-filter=include-files.lua"
+                "--lua-filter=include-files.lua"
-        ])
+            ])
 
     page.body = pypandoc.convert_text(
         latex,
@@ -146,53 +185,78 @@ def get_page_data(path, isPreload=False):
 
     return page
 
+# Do stuff to the circuit's pcb
+def save_circuit_svg(filepath, outpath, name):
 
-def render_posts(path):
+    tree = ET.parse(filepath)
+    root = tree.getroot()
+
+    # Extract current width/height (in pixels)
+    width_px = float(root.get("width", 0))
+    height_px = float(root.get("height", 0))
+
+    DPI = 300
+
+    # Convert px to mm
+    width_mm = (width_px * 25.4) / DPI > 15
+    height_mm = (height_px * 25.4) / DPI
+
+    # Set new width/height in mm
+    root.set("width", f"{width_px}mm")
+    root.set("height", f"{height_px}mm")
+
+    os.makedirs(outpath, exist_ok = True)
+
+
+    tree.write(f"{outpath}/{name}")
+
+
+# combine HTML & data with Jinja templates
+def render_posts(path, output_path=OUTPUT_D):
     name = Path(path).stem
-    template = env.select_template([f"{name}.jinja", "post.jinja"])
+   
-
     for filename in os.listdir(path):
+        file_path = os.path.join(path, filename)
+
         if filename.endswith(".md"):
-            post_path = os.path.join(path, filename)
+            page = get_page_data(file_path)
-            page = get_page_data(post_path)
+            render_single_file(page, file_path, f"{output_path}/{name}", name)
-            render_single_file(template, page, post_path, f"{OUTPUT_D}/{name}", name)
+        elif os.path.isdir(file_path):   
-
+            render_posts(file_path, f"{output_path}/{name}")
+        elif filename.endswith(".svg"):
+            save_circuit_svg(file_path, f"{output_path}/{name}", filename)
+        else:
+            print("doing nothing with", filename)
 
+# Pre-load before compiling
 def preload_documents():
     print("preload any needed data")
-    now = datetime.now()
+    documents["meta"] = {"now": now.strftime("%d %B %Y")}
-    documents["meta"] = {}
-    documents["meta"]["now"] = now.strftime("%d %B %Y")
 
     for subdir in os.listdir(CONTENT_D):
         path = os.path.join(CONTENT_D, subdir)
+        print(path)
 
         if os.path.isdir(path):
             name = Path(path).stem
-            if name not in documents:
+            documents.setdefault(name, [])
-                documents[name] = []
 
-            files = os.listdir(path)
+            for filename in sorted(os.listdir(path)):
-            files.sort()
-            for filename in files:
                 if filename.endswith(".md"):
-                    post_path = os.path.join(path, filename)
+                    documents[name].append(get_page_data(os.path.join(path, filename), isPreload=True))
-
-                    documents[name].append(get_page_data(post_path, isPreload=True))
 
         elif Path(path).suffix == '.md':
             documents[Path(path).stem] = get_page_data(path, isPreload=True)
 
 
 def copy_assets():
-    if os.path.exists("dist/assets"):
+    if os.path.exists(OUT_ASSETS):
-        shutil.rmtree("dist/assets")
+        shutil.rmtree(OUT_ASSETS)
 
-    shutil.copytree("src/assets", "dist/assets")
+    shutil.copytree(SRC_ASSETS, OUT_ASSETS)
 
 
 def get_inventory():
-
     with open("src/content/component-inventory.csv") as f:
         documents['inventory'] = []
         for line in csv.DictReader(
@@ -213,19 +277,14 @@ def main():
     get_inventory()
     preload_documents()
 
-
-
-    print("render the content")
     for subdir in os.listdir(CONTENT_D):
         path = os.path.join(CONTENT_D, subdir)
 
         if os.path.isdir(path):
+            print("rendering posts", path)
             render_posts(path)
         elif Path(path).suffix == '.md':
-            template = env.select_template(
+            render_single_file(get_page_data(path), path, OUTPUT_D)
-                [f"{Path(path).stem}.jinja", "post.jinja"])
-            page = get_page_data(path)
-            render_single_file(template, page, path, OUTPUT_D)
         elif Path(path).suffix in [".csv"]:
             print("not compiling this file!")