fixes for speed (issue #992), and added a sitemap generator which aut…

…omatically regenerates the sitemap when the app is restarted (issue #1276 and #1020)

app.py

@@ -1,18 +1,16 @@
-from flask import Flask, render_template, request, redirect, abort, render_template_string, send_from_directory
+from flask import Flask, render_template, request, redirect, abort, render_template_string, send_from_directory, send_file
 from flask_assets import Environment, Bundle
 from datetime import datetime
-from functions import build_data_dict, fetch_contributions_for_the_single_contributor, generate_table_of_contents, get_breadcrumbs, find_related_articles, calculate_reading_time, fetch_meta_data, recently_published
+from functions import build_data_dict, fetch_contributions_for_the_single_contributor, generate_table_of_contents, get_breadcrumbs, find_related_articles, calculate_reading_time, fetch_meta_data, recently_published, generate_sitemap
 import os
 from models import db, articles, Contributors, blogs, Topics
 from html_parser import htmlize, r_to_html_plaintext
 from redirectstsh import setup_redirects
 from utils import get_git_commit_hash
 import redirects_config
-from flask_sitemap import Sitemap
 
 # Initialize App
 app = Flask(__name__, static_url_path='/static')
-ext = Sitemap(app=app)
 
 @app.context_processor
 def inject_git_commit_hash():
@@ -23,6 +21,9 @@ def inject_git_commit_hash():
 db_path = os.path.abspath(os.path.join(os.path.dirname(__file__), db_filename))
 db_uri = f'sqlite:///{db_path}'  
 
+# Root URL
+base_url = 'https://flask.tilburgsciencehub.com'
+
 # Database configuration
 app.config['SQLALCHEMY_DATABASE_URI'] = db_uri
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
@@ -43,15 +44,17 @@ def inject_git_commit_hash():
 
 # Build & Register SCSS Bundle
 assets = Environment(app)
-
 scss_bundle = Bundle(
     'scss/bootstrap.scss',  
     output='css/main.css',
     filters='scss',
 )
-
 assets.register('scss_all', scss_bundle)
 
+# Create data dict
+with app.app_context():
+    data_dict = build_data_dict(Topics, articles)
+
 # Custom filter for dates
 @app.template_filter('formatdate')
 def formatdate(value, format="%Y-%m-%d"):
@@ -68,15 +71,15 @@ def home():
     data_object = {'title' : 'Home'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     recent_articles = recently_published(articles, Topics)
 
     return render_template('index.html', assets=assets, data_dict=data_dict, meta_data=meta_data, recent_articles=recent_articles)
 
 # Single Example
 @app.route('/examples/<article_path>')
 def example(article_path):
-    data_dict = build_data_dict(Topics, articles)
+
     breadcrumbs = get_breadcrumbs()
     current_url = request.url
     article = None
@@ -92,7 +95,7 @@ def example(article_path):
 # Single Blog
 @app.route('/blog/<blog_path>')
 def blogs_single(blog_path):
-    data_dict = build_data_dict(Topics, articles)
+
     breadcrumbs = get_breadcrumbs()
     blog_query = None
     blog_data = None
@@ -109,37 +112,37 @@ def blogs_single(blog_path):
 # List Topics
 @app.route('/topics')
 def topics_list():
-    data_dict = build_data_dict(Topics, articles)
+
     return render_template('topics-list.html', assets=assets, data_dict=data_dict)
 
 # Still needs metadata!
 # First Level Topic
 @app.route('/topics/<first_level_topic_path>/')
 def topics_first_level(first_level_topic_path):
-    data_dict = build_data_dict(Topics, articles)
+
 
     return render_template('first-level-topic.html', assets=assets, data_dict=data_dict, topic_path=first_level_topic_path)
 
 # Still needs metadata!
 # Second Level Topic
 @app.route('/topics/<first_level_topic_path>/<second_level_topic_path>/')
 def topics_second_level(first_level_topic_path,second_level_topic_path):
-    data_dict = build_data_dict(Topics, articles)
+
 
     return render_template('second-level-topic.html', assets=assets, data_dict=data_dict, topic_path=first_level_topic_path, sec_level_topic_path=second_level_topic_path)
 
 # Still needs metadata!
 # Third Level Topic
 @app.route('/topics/<first_level_topic_path>/<second_level_topic_path>/<third_level_topic_path>/')
 def topics_third_level(first_level_topic_path,second_level_topic_path, third_level_topic_path):
-    data_dict = build_data_dict(Topics, articles)
+
 
     return render_template('third-level-topic.html', assets=assets, data_dict=data_dict, topic_path=first_level_topic_path, sec_level_topic_path=second_level_topic_path, third_level_topic_path=third_level_topic_path)
 
 # Single Article (Topic)
 @app.route('/topics/<first_level_topic_path>/<second_level_topic_path>/<third_level_topic_path>/<article_path>/')
 def topic_single(first_level_topic_path, second_level_topic_path, third_level_topic_path, article_path):
-    data_dict = build_data_dict(Topics, articles)
+
     breadcrumbs = get_breadcrumbs()
     current_url = request.url
     article = None
@@ -173,7 +176,7 @@ def examples():
     data_object = {'title' : 'Examples'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     print(data_dict)
     return render_template('examples-list.html', assets=assets, data_dict=data_dict, meta_data=meta_data)
 
@@ -185,7 +188,7 @@ def blog():
     data_object = {'title' : 'Blog'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     blogs_data = blogs.query.all()
 
     # Lus door de lijst en formatteer de datum in elk item
@@ -214,7 +217,7 @@ def about():
     data_object = {'title' : 'About Us'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     return render_template('about.html', assets=assets, data_dict=data_dict, meta_data=meta_data)
 
 # Contribute
@@ -235,7 +238,7 @@ def search():
     data_object = {'title' : 'Search'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     return render_template('search.html', assets=assets, data_dict=data_dict, meta_data=meta_data)
 
 # Contributors
@@ -246,15 +249,15 @@ def contributors():
     data_object = {'title' : 'Contributors'}
     meta_data = fetch_meta_data(data_object)
 
-    data_dict = build_data_dict(Topics, articles)
+
     contributors_list = Contributors.query.all()
     return render_template('contributors-list.html', assets=assets, data_dict=data_dict, contributors_list=contributors_list, meta_data=meta_data)
 
 # Still needs metadata!
 # Single Contributor
 @app.route('/contributors/<contributor_path>')
 def contributor(contributor_path):
-    data_dict = build_data_dict(Topics, articles)
+
     contributor_single = Contributors.query.filter_by(
         path=contributor_path).first()
 
@@ -265,13 +268,10 @@ def contributor(contributor_path):
 
     return render_template('contributors-single.html', assets=assets, data_dict=data_dict, contributor_single=contributor_single, contributions=contributions)
 
-# Redirects
-setup_redirects(app)
-
 # Custom redirects
 @app.route('/<path:path>', methods=['GET'])
 def handle_redirect(path):
-    data_dict = build_data_dict(Topics, articles)
+
     redirect_info = redirects_config.REDIRECTS.get(f"/{path}")
     if redirect_info:
         new_url = redirect_info["url"]
@@ -295,13 +295,25 @@ def robots_txt():
 # Error Handler 404
 @app.errorhandler(404)
 def page_not_found(e):
-    data_dict = build_data_dict(Topics, articles)
+
     return render_template('404.html', assets=assets, data_dict=data_dict), 404
 
-# Sitemap
 @app.route('/sitemap.xml')
 def sitemap():
-    return ext.sitemap()
+    # Pad naar de gegenereerde sitemap.xml
+    sitemap_path = 'sitemap.xml'
+
+    # Return het sitemap.xml bestand
+    return send_file(sitemap_path, mimetype='application/xml')
+
+
+# Sitemap without redirects
+with app.app_context():
+    sitemap = generate_sitemap(app, data_dict, base_url=base_url)
+
+# Redirects
+setup_redirects(app)
 
+# Run App
 if __name__ == '__main__':
     app.run(debug=True)

functions.py

@@ -7,6 +7,8 @@
 from flask import request
 import math 
 from collections import defaultdict
+import json
+import xml.etree.ElementTree as ET
 
 nltk.download('stopwords')
 stop_words = set(stopwords.words('english'))
@@ -59,35 +61,40 @@ def serialize_article(article):
             'title': article.title,
             'path': article.path,
             'description': article.description,
-            'reading_time': calculate_reading_time(article.content)
+            'reading_time': calculate_reading_time(article.content),
+            'date': article.date,
+            'draft': article.draft
+        }
+
+    def serialize_topic(topic, level, parent, articles):
+        return {
+            'id': topic.id,
+            'title': topic.title,
+            'path': topic.path,
+            'parent': topic.parent,
+            'level': topic.level,
+            'draft': topic.draft,
+            'childtopics': build_structure(level + 1, topic.id, articles),
+            'articles': [serialize_article(article) for article in articles.query.filter_by(parent=topic.id).all()]
         }
 
     def build_structure(level, parent, articles):
         if level not in topic_dict:
             return []
         return [
-            {
-                'id': topic.id,
-                'title': topic.title,
-                'path': topic.path,
-                'parent': topic.parent,
-                'level': topic.level,
-                'draft': topic.draft,
-                'childtopics': build_structure(level + 1, topic.id, articles),
-                'articles': [serialize_article(article) for article in articles.query.filter_by(parent=topic.id).all()]
-            }
+            serialize_topic(topic, level, parent, articles)
             for topic in topic_dict[level][parent]
         ]
 
     articles_examples = articles.query.filter_by(type='examples').all()
-    for article in articles_examples:
-        article.reading_time = calculate_reading_time(article.content)
-
+    serialized_articles_examples = [serialize_article(article) for article in articles_examples]
+
     data_dict['topics'] = build_structure(1, 1, articles)
-    data_dict['examples'] = articles_examples
+    data_dict['examples'] = serialized_articles_examples
 
     return data_dict
 
+
 # Recursively find the full path of a topic by its parent ID
 # Parameters:
 # - parent_topic_id: Integer ID of the parent topic
@@ -306,3 +313,57 @@ def fetch_contributions_for_the_single_contributor(Contributor, Articles, Topics
             "path": full_path
         })
     return contributions_with_full_path
+
+def add_url_element(urlset, loc, date=None, priority="0.8", changefreq="monthly"):
+    url = ET.SubElement(urlset, "url")
+
+    loc_element = ET.SubElement(url, "loc")
+    loc_element.text = loc
+
+    priority_element = ET.SubElement(url, "priority")
+    priority_element.text = priority
+
+    changefreq_element = ET.SubElement(url, "changefreq")
+    changefreq_element.text = changefreq
+
+    if date:
+        lastmod_element = ET.SubElement(url, "lastmod")
+        lastmod_element.text = str(date)
+
+def generate_sitemap(app, data, base_url="https://www.example.com"):
+    urlset = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
+
+    def process_topics(topics, parent_path=""):
+        for topic in topics:
+            if topic['draft'] != 'true':
+                full_path = f"{parent_path}/{topic['path']}".strip("/")
+                add_url_element(urlset, f"{base_url}/topics/{full_path}")
+                process_topics(topic.get('childtopics', []), full_path)
+                for article in topic.get('articles', []):
+                    if article['draft'] != 'true':
+                        article_path = f"{full_path}/{article['path']}".strip("/")
+                        add_url_element(urlset, f"{base_url}/topics/{article_path}", date = article['date'])
+
+    # Process topics and articles
+    process_topics(data['topics'])
+
+    # Process examples
+    for example in data.get('examples', []):
+        if example['draft'] != 'true':
+            example_path = f"{base_url}/examples/{example['path']}".strip("/")
+            add_url_element(urlset, example_path, date=example.get('date'))
+
+    # Process blogs
+
+    # Non dynamic routes
+    rules = list(app.url_map.iter_rules())
+    if not rules:
+        print("No routes were registered.")
+    else:
+        for rule in app.url_map.iter_rules():
+            if "GET" in rule.methods and "<" not in rule.rule and len(rule.arguments) == 0:
+                if "/building-blocks" not in rule.rule and "/tutorials" not in rule.rule:
+                    add_url_element(urlset, f"{base_url}{rule.rule}", priority="0.5")
+
+    tree = ET.ElementTree(urlset)
+    tree.write("sitemap.xml", encoding="utf-8", xml_declaration=True)