From 9a93d5c60323b32bfad31e8e8921e8fae198b443 Mon Sep 17 00:00:00 2001 From: vitrinekast Date: Sun, 16 Feb 2025 11:08:10 +0100 Subject: [PATCH] make script faster --- app.py | 187 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 123 insertions(+), 64 deletions(-) diff --git a/app.py b/app.py index b4e4fb3..ed8d3a2 100644 --- a/app.py +++ b/app.py @@ -10,8 +10,10 @@ import frontmatter from slugify import slugify import pypandoc -# TODO make newsletter URL's absolute to klank.school +import xml.etree.ElementTree as ET + +# TODO make newsletter URL's absolute to klank.school env = Environment( loader=PackageLoader("src"), autoescape=select_autoescape() @@ -19,17 +21,12 @@ env = Environment( CONTENT_D = os.path.abspath("src/content") OUTPUT_D = "dist" +OUT_ASSETS = "dist/assets" +SRC_ASSETS = "src/assets" documents = {} +now = datetime.now() -def listDocuments(params): - param = params.split(" ") - - template = env.select_template([f"snippets/list-documents.jinja"]) - html = template.render(documents=documents, layout=param[0], type=param[1]) - - return html - - +# Utils def getParam(params, index): if len(params) > index: return params[index] @@ -37,9 +34,18 @@ def getParam(params, index): return False +# jinja filter that can list documents +def listDocuments(params): + param = params.split(" ") + template = env.select_template(["snippets/list-documents.jinja"]) + html = template.render(documents=documents, layout=param[0], type=param[1]) + + return html + +# jinja filter that can list events def listEvents(params): param = params.split(" ") - tag=getParam(param, 1) + tag = getParam(param, 1) if "events" not in documents: return "" @@ -52,20 +58,21 @@ def listEvents(params): else: events = documents["events"] - template = env.select_template([f"snippets/list-events.jinja"]) + template = env.select_template(["snippets/list-events.jinja"]) html = template.render(events=events, filter=param[0], tag=getParam(param, 1)) return html - +# jinja filter to make a slug out of a stirng def slugify_filter(value): return slugify(value) - +# jinja filter for date formatting def prettydate(value, format='%d/%m/%Y'): return datetime.fromtimestamp(int(value)).strftime(format) +# jinja filter to replace shortcodes in HTML def shortcode_filter(value): shortcode_callbacks = { @@ -91,12 +98,9 @@ env.filters["shortcode"] = shortcode_filter env.filters["slugify"] = slugify_filter env.filters["prettydate"] = prettydate - -def render_single_file(template, page, path, dist, name = False): - modifier = False - - print(f"rendering: {path}") - +# translate a single file into HTML +def render_single_file(page, path, dist, name = False): + template = env.select_template([f"{name}.jinja", "post.jinja"]) html = template.render(documents=documents, page=page, name=name) name = Path(path).stem @@ -107,30 +111,65 @@ def render_single_file(template, page, path, dist, name = False): output_file.write(html) +# find a pre-rendered page +def get_existing_page(path, slug): + stem = Path(path).stem; + folder = os.path.basename(os.path.dirname(path)) + + if stem == "index" and folder != "content": + folder = Path(path).parent.parent.name + + if slug in documents: + return documents[slug] + + if folder == "content": + return False + + for doc in documents[folder]: + if doc["slug"] == slug: + return doc + + return False + +# build a slug including the folder +def get_slug(path, folder, filename): + if folder == "content": + return slugify(filename) + else: + return slugify(f"{folder}/{filename}") + +# compile markdown into cited HTML def get_page_data(path, isPreload=False): + filename = Path(path).stem + folder = os.path.basename(os.path.dirname(path)) + slug = get_slug(path, folder, filename) + + prerendered = get_existing_page(path, slug) + + if prerendered: + return prerendered page = frontmatter.load(path) - page['slug'] = slugify(filename) + page['slug'] = slug page.filename = filename - page.folder = os.path.basename(os.path.dirname(path)) + page.folder = folder + latex = page.content if "start_datetime" in page: - now = datetime.now() page["has_passed"] = datetime.fromtimestamp(page["start_datetime"]) < now - - - - latex = pypandoc.convert_text( - page.content, - to='md', - format='md', - extra_args=[ - "-N", - "--section-divs", - "--lua-filter=include-files.lua" - ]) + + if ".include" in page.content: + latex = pypandoc.convert_text( + page.content, + to='md', + format='md', + extra_args=[ + "-N", + "--section-divs", + "--lua-filter=include-files.lua" + ]) page.body = pypandoc.convert_text( latex, @@ -146,53 +185,78 @@ def get_page_data(path, isPreload=False): return page +# Do stuff to the circuit's pcb +def save_circuit_svg(filepath, outpath, name): -def render_posts(path): + tree = ET.parse(filepath) + root = tree.getroot() + + # Extract current width/height (in pixels) + width_px = float(root.get("width", 0)) + height_px = float(root.get("height", 0)) + + DPI = 300 + + # Convert px to mm + width_mm = (width_px * 25.4) / DPI > 15 + height_mm = (height_px * 25.4) / DPI + + # Set new width/height in mm + root.set("width", f"{width_px}mm") + root.set("height", f"{height_px}mm") + + os.makedirs(outpath, exist_ok = True) + + + tree.write(f"{outpath}/{name}") + + +# combine HTML & data with Jinja templates +def render_posts(path, output_path=OUTPUT_D): name = Path(path).stem - template = env.select_template([f"{name}.jinja", "post.jinja"]) - + for filename in os.listdir(path): + file_path = os.path.join(path, filename) + if filename.endswith(".md"): - post_path = os.path.join(path, filename) - page = get_page_data(post_path) - render_single_file(template, page, post_path, f"{OUTPUT_D}/{name}", name) - + page = get_page_data(file_path) + render_single_file(page, file_path, f"{output_path}/{name}", name) + elif os.path.isdir(file_path): + render_posts(file_path, f"{output_path}/{name}") + elif filename.endswith(".svg"): + save_circuit_svg(file_path, f"{output_path}/{name}", filename) + else: + print("doing nothing with", filename) +# Pre-load before compiling def preload_documents(): print("preload any needed data") - now = datetime.now() - documents["meta"] = {} - documents["meta"]["now"] = now.strftime("%d %B %Y") + documents["meta"] = {"now": now.strftime("%d %B %Y")} for subdir in os.listdir(CONTENT_D): path = os.path.join(CONTENT_D, subdir) + print(path) if os.path.isdir(path): name = Path(path).stem - if name not in documents: - documents[name] = [] + documents.setdefault(name, []) - files = os.listdir(path) - files.sort() - for filename in files: + for filename in sorted(os.listdir(path)): if filename.endswith(".md"): - post_path = os.path.join(path, filename) - - documents[name].append(get_page_data(post_path, isPreload=True)) + documents[name].append(get_page_data(os.path.join(path, filename), isPreload=True)) elif Path(path).suffix == '.md': documents[Path(path).stem] = get_page_data(path, isPreload=True) def copy_assets(): - if os.path.exists("dist/assets"): - shutil.rmtree("dist/assets") + if os.path.exists(OUT_ASSETS): + shutil.rmtree(OUT_ASSETS) - shutil.copytree("src/assets", "dist/assets") + shutil.copytree(SRC_ASSETS, OUT_ASSETS) def get_inventory(): - with open("src/content/component-inventory.csv") as f: documents['inventory'] = [] for line in csv.DictReader( @@ -213,19 +277,14 @@ def main(): get_inventory() preload_documents() - - - print("render the content") for subdir in os.listdir(CONTENT_D): path = os.path.join(CONTENT_D, subdir) if os.path.isdir(path): + print("rendering posts", path) render_posts(path) elif Path(path).suffix == '.md': - template = env.select_template( - [f"{Path(path).stem}.jinja", "post.jinja"]) - page = get_page_data(path) - render_single_file(template, page, path, OUTPUT_D) + render_single_file(get_page_data(path), path, OUTPUT_D) elif Path(path).suffix in [".csv"]: print("not compiling this file!")