Fix toctrees and reference pages

doc/generate_reference.py

@@ -15,9 +15,12 @@
 from param import concrete_descendents
 
 import lumen
+import lumen.sources.duckdb
 import lumen.sources.intake
 import lumen.sources.intake_sql
 
+from lumen.ai.actor import Actor
+from lumen.ai.agents import Agent
 from lumen.base import MultiTypeComponent
 from lumen.dashboard import Auth, Config, Defaults
 from lumen.filters import Filter
@@ -28,7 +31,7 @@
 from lumen.variables import Variable
 from lumen.views import View
 
-bases = [Config, Variable, Pipeline, Source, Filter, Transform, View, Layout, Auth, Defaults]
+bases = [Agent, Config, Variable, Pipeline, Source, Filter, Transform, View, Layout, Auth, Defaults]
 
 BASE_PATH = pathlib.Path(lumen.__file__).parent.parent
 REFERENCE_PATH = BASE_PATH / 'doc' / 'reference'
@@ -128,7 +131,7 @@ def generate_param_docs(base, component, gutter=3, margin=0):
         pobj = component.param[pname]
         ptype = type(pobj)
         pbadge = f'<span style="font-size: 1.25em;">{{bdg-dark}}`{pname}`'
-        if pname in component._required_keys:
+        if pname in getattr(component, '_required_keys', []):
             pbadge += '&emsp;&emsp;&emsp;{bdg-warning}`required`'
         pbadge += '</span>'
         pitems = []
@@ -189,7 +192,7 @@ def generate_automethods(component):
 
 def generate_page(base, component):
     title = f'# {component.__name__}'
-    if base is not component:
+    if base is not component and issubclass(component, MultiTypeComponent):
         ctype = getattr(component, f'{base.__name__.lower()}_type')
         title += f'&nbsp;&nbsp;{{bdg-primary}}`type: {ctype}`'
     title += '\n'
@@ -230,9 +233,9 @@ def generate_grid(base, gridspec='2 2 3 3', gutter=3, margin=0, rel=''):
             description = component.__doc__.split('\n')[1].strip()
         else:
             description = f'{name} {base.__name__}'
-        ctype = getattr(component, f'{base.__name__.lower()}_type')
-        badge = f'   {{bdg-primary}}`type: {ctype}`'
-        page += f':::{{grid-item-card}} {name} {badge}\n:link: {rel}{name}.html\n:shadow: md\n\n{description}\n:::\n\n'
+        ctype = getattr(component, f'{base.__name__.lower()}_type', None)
+        badge = f'   {{bdg-primary}}`type: {ctype}`' if ctype else ''
+        page += f':::{{grid-item-card}} {name}{badge}\n:link: {rel}{name}.html\n:shadow: md\n\n{description}\n:::\n\n'
     page += '::::'
     return page
 
@@ -245,16 +248,28 @@ def generate_multi_component_pages(base):
     index = generate_component_index(base)
     component_path = REFERENCE_PATH /  base.__name__.lower()
     component_path.mkdir(exist_ok=True, parents=True)
-    with open(component_path / 'index.md', 'w') as f:
-        f.write(index)
+    toc = ''
     for component in concrete_descendents(base).values():
         if component.__name__.startswith('_'):
             continue
         page = generate_page(base, component)
-        path = REFERENCE_PATH /  base.__name__.lower() / f'{component.__name__}.md'
+        name = component.__name__
+        path = REFERENCE_PATH /  base.__name__.lower() / f'{name}.md'
         print(f'Writing {path}.')
         with open(path, 'w') as f:
             f.write(page)
+        toc += f'{name}\n'
+    index += f"""
+
+```{{toctree}}
+---
+hidden: true
+---
+{toc[:-1]}
+```
+"""
+    with open(component_path / 'index.md', 'w') as f:
+        f.write(index)
     return page
 
 def generate_single_component_page(component):
@@ -265,29 +280,40 @@ def generate_single_component_page(component):
 
 def write_index(bases):
     page = '# Reference\n\n'
+    toc = ''
     for base in bases:
+        name = base.__name__
         if base.__doc__:
             description = base.__doc__.split("\n")[1].strip()
         else:
             description = None
-        if issubclass(base, MultiTypeComponent):
-            page += f'## [`{base.__name__}`]({base.__name__.lower()}/index)\n\n'
+        if issubclass(base, (Actor, MultiTypeComponent)):
+            page += f'## [`{name}`]({name.lower()}/index)\n\n'
             if description:
                 page += f'{description}\n\n'
             page += generate_grid(base, rel=f'{base.__name__.lower()}/')
             page += '\n\n'
         else:
-            page += f'## [`{base.__name__}`]({base.__name__})\n\n'
+            page += f'## [`{name}`]({name})\n\n'
             if description:
                 page += f'{description}\n\n'
+        toc += f'{name} <{name.lower()}/index>\n'
+    page += f"""
+```{{toctree}}
+---
+hidden: true
+---
+{toc[:-1]}
+```
+"""
     path = REFERENCE_PATH / 'index.md'
     with open(path, 'w') as f:
         f.write(page)
 
 if __name__ == '__main__':
     REFERENCE_PATH.mkdir(exist_ok=True, parents=True)
     for component in bases:
-        if issubclass(component, MultiTypeComponent):
+        if issubclass(component, (Actor, MultiTypeComponent)):
             generate_multi_component_pages(component)
         else:
             generate_single_component_page(component)

doc/getting_started/index.md

@@ -12,6 +12,13 @@ This _Getting Started_ guide will go through the installation of Lumen, how to b
 Install Lumen in a few easy steps
 :::
 
+::::
+
+## AI
+
+::::{grid} 1 2 2 3
+:gutter: 1 1 1 2
+
 :::{grid-item-card} {octicon}`zap;2em;sd-mr-1` Using Lumen AI
 :link: using_lumen_ai
 :link-type: doc
@@ -26,6 +33,13 @@ How to explore your data with Lumen AI
 Configure Lumen AI to use custom Coordinators, Agents, and LLM providers
 :::
 
+::::
+
+## Core
+
+::::{grid} 1 2 2 3
+:gutter: 1 1 1 2
+
 :::{grid-item-card} {octicon}`tools;2em;sd-mr-1` Build a dashboard
 :link: build_dashboard
 :link-type: doc
@@ -54,7 +68,8 @@ Get an overview of the core concepts of Lumen
 hidden: true
 ---
 installation
-lumen_ai
+using_lumen_ai
+configuring_lumen_ai
 build_dashboard
 core_concepts
 pipelines

doc/how_to/index.md

@@ -43,7 +43,7 @@ Learn to create and use template prompts for influencing the AI's responses.
 :::
 
 ::::
-2
+
 ## Data Intake
 
 ::::{grid} 1 2 2 3

doc/index.md

@@ -26,7 +26,7 @@ Install Lumen in a few easy steps
 :::
 
 :::{grid-item-card} {octicon}`zap;2em;sd-mr-1` Exploration with Lumen AI
-:link: getting_started/lumen_ai
+:link: getting_started/using_lumen_ai
 :link-type: doc
 
 How to explore your data with Lumen AI.

lumen/ai/controls.py

@@ -4,7 +4,6 @@
 import pandas as pd
 import param
 
-from markitdown import MarkItDown
 from panel.layout import (
     Column, FlexBox, Row, Tabs,
 )
@@ -123,9 +122,8 @@ class SourceControls(Viewer):
 
     def __init__(self, **params):
         super().__init__(**params)
-
-        self._markitdown = MarkItDown()
         self.tables_tabs = Tabs(sizing_mode="stretch_width")
+        self._markitdown = None
         self._file_input = FileDropper(
             height=100,
             multiple=self.param.multiple,
@@ -278,6 +276,9 @@ def _add_document(
         file: io.BytesIO,
         document_controls: DocumentControls
     ):
+        if self._markitdown is None:
+            from markitdown import MarkItDown
+            self._markitdown = MarkItDown()
         text = self._markitdown.convert_stream(
             file, file_extension=document_controls.extension
         ).text_content

pixi.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lumen"
-channels = ["pyviz/label/dev", "bokeh", "conda-forge"]
+channels = ["pyviz/label/dev", "bokeh", "conda-forge", "conda-forge/label/markitdown_dev"]
 platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"]
 
 [tasks]
@@ -10,11 +10,11 @@ install = 'python -m pip install --no-deps --disable-pip-version-check -e .'
 PYTHONIOENCODING = "utf-8"
 
 [environments]
-test-310 = ["py310", "test-core", "test"]
-test-311 = ["py311", "test-core", "test"]
-test-312 = ["py312", "test-core", "test"]
+test-310 = ["py310", "test-core", "test", "ai", "sql"]
+test-311 = ["py311", "test-core", "test", "ai", "sql"]
+test-312 = ["py312", "test-core", "test", "ai", "sql"]
 test-core = ["py312", "test-core"]
-docs = ["py311", "doc"]
+docs = ["py311", "doc", "ai", "sql"]
 build = ["py311", "build"]
 lint = ["py311", "lint"]
 
@@ -40,6 +40,28 @@ python = "3.11.*"
 [feature.py312.dependencies]
 python = "3.12.*"
 
+[feature.ai.dependencies]
+duckdb = "*"
+griffe = "*"
+instructor = ">=1.4.3"
+markitdown = "*"
+nbformat = "*"
+openai = "*"
+pyarrow = "*"
+pydantic = ">=2.8.0"
+pydantic-extra-types = "*"
+
+[feature.ai-local.dependencies]
+huggingface_hub = "*"
+
+[feature.ai-llama.dependencies]
+llama-cpp-python = ">=0.3.0"
+
+[feature.sql.dependencies]
+duckdb = "*"
+intake-sql = "*"
+sqlalchemy = "*"
+
 # =============================================
 # =================== TESTS ===================
 # =============================================
@@ -57,44 +79,12 @@ dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests
 test-unit = 'pytest lumen/tests -n logical --dist loadgroup'
 
 [feature.test.dependencies]
+dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests
 dask-core = "*"
 fastparquet = "*"
 matplotlib-base = ">=3.4" # Ubuntu + Python 3.9 installs old version matplotlib (3.3.2)
 msgpack-python = "*"
 toolz = "*"
-dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests
-# sql
-intake-sql = "*"
-python-duckdb = "*"
-sqlalchemy = "*"
-# ai
-datashader = "*"
-duckdb = "*"
-griffe = "*"
-instructor = ">=1.4.3"
-nbformat = "*"
-openai = "*"
-pyarrow = "*"
-pydantic = ">=2.8.0"
-
-[feature.test.pypi-dependencies]
-pydantic_extra_types = "*"
-
-# [feature.ai.dependencies]
-# datashader = "*"
-# python-duckdb = "*"
-# instructor = ">=1.4.3"
-# nbformat = "*"
-# openai = "*"
-# pyarrow = "*"
-# pydantic = ">=2.8.0"
-#
-
-# [feature.ai-local.dependencies]
-# huggingface_hub = "*"
-#
-# [feature.ai-llama.dependencies]
-# llama-cpp-python = ">=0.3.0"
 
 # =============================================
 # =================== DOCS ====================