diff --git a/doc/generate_reference.py b/doc/generate_reference.py index 8db1e2de..60d77450 100644 --- a/doc/generate_reference.py +++ b/doc/generate_reference.py @@ -15,9 +15,12 @@ from param import concrete_descendents import lumen +import lumen.sources.duckdb import lumen.sources.intake import lumen.sources.intake_sql +from lumen.ai.actor import Actor +from lumen.ai.agents import Agent from lumen.base import MultiTypeComponent from lumen.dashboard import Auth, Config, Defaults from lumen.filters import Filter @@ -28,7 +31,7 @@ from lumen.variables import Variable from lumen.views import View -bases = [Config, Variable, Pipeline, Source, Filter, Transform, View, Layout, Auth, Defaults] +bases = [Agent, Config, Variable, Pipeline, Source, Filter, Transform, View, Layout, Auth, Defaults] BASE_PATH = pathlib.Path(lumen.__file__).parent.parent REFERENCE_PATH = BASE_PATH / 'doc' / 'reference' @@ -128,7 +131,7 @@ def generate_param_docs(base, component, gutter=3, margin=0): pobj = component.param[pname] ptype = type(pobj) pbadge = f'{{bdg-dark}}`{pname}`' - if pname in component._required_keys: + if pname in getattr(component, '_required_keys', []): pbadge += '   {bdg-warning}`required`' pbadge += '' pitems = [] @@ -189,7 +192,7 @@ def generate_automethods(component): def generate_page(base, component): title = f'# {component.__name__}' - if base is not component: + if base is not component and issubclass(component, MultiTypeComponent): ctype = getattr(component, f'{base.__name__.lower()}_type') title += f'  {{bdg-primary}}`type: {ctype}`' title += '\n' @@ -230,9 +233,9 @@ def generate_grid(base, gridspec='2 2 3 3', gutter=3, margin=0, rel=''): description = component.__doc__.split('\n')[1].strip() else: description = f'{name} {base.__name__}' - ctype = getattr(component, f'{base.__name__.lower()}_type') - badge = f' {{bdg-primary}}`type: {ctype}`' - page += f':::{{grid-item-card}} {name} {badge}\n:link: {rel}{name}.html\n:shadow: md\n\n{description}\n:::\n\n' + ctype = getattr(component, f'{base.__name__.lower()}_type', None) + badge = f' {{bdg-primary}}`type: {ctype}`' if ctype else '' + page += f':::{{grid-item-card}} {name}{badge}\n:link: {rel}{name}.html\n:shadow: md\n\n{description}\n:::\n\n' page += '::::' return page @@ -245,16 +248,28 @@ def generate_multi_component_pages(base): index = generate_component_index(base) component_path = REFERENCE_PATH / base.__name__.lower() component_path.mkdir(exist_ok=True, parents=True) - with open(component_path / 'index.md', 'w') as f: - f.write(index) + toc = '' for component in concrete_descendents(base).values(): if component.__name__.startswith('_'): continue page = generate_page(base, component) - path = REFERENCE_PATH / base.__name__.lower() / f'{component.__name__}.md' + name = component.__name__ + path = REFERENCE_PATH / base.__name__.lower() / f'{name}.md' print(f'Writing {path}.') with open(path, 'w') as f: f.write(page) + toc += f'{name}\n' + index += f""" + +```{{toctree}} +--- +hidden: true +--- +{toc[:-1]} +``` +""" + with open(component_path / 'index.md', 'w') as f: + f.write(index) return page def generate_single_component_page(component): @@ -265,21 +280,32 @@ def generate_single_component_page(component): def write_index(bases): page = '# Reference\n\n' + toc = '' for base in bases: + name = base.__name__ if base.__doc__: description = base.__doc__.split("\n")[1].strip() else: description = None - if issubclass(base, MultiTypeComponent): - page += f'## [`{base.__name__}`]({base.__name__.lower()}/index)\n\n' + if issubclass(base, (Actor, MultiTypeComponent)): + page += f'## [`{name}`]({name.lower()}/index)\n\n' if description: page += f'{description}\n\n' page += generate_grid(base, rel=f'{base.__name__.lower()}/') page += '\n\n' else: - page += f'## [`{base.__name__}`]({base.__name__})\n\n' + page += f'## [`{name}`]({name})\n\n' if description: page += f'{description}\n\n' + toc += f'{name} <{name.lower()}/index>\n' + page += f""" +```{{toctree}} +--- +hidden: true +--- +{toc[:-1]} +``` +""" path = REFERENCE_PATH / 'index.md' with open(path, 'w') as f: f.write(page) @@ -287,7 +313,7 @@ def write_index(bases): if __name__ == '__main__': REFERENCE_PATH.mkdir(exist_ok=True, parents=True) for component in bases: - if issubclass(component, MultiTypeComponent): + if issubclass(component, (Actor, MultiTypeComponent)): generate_multi_component_pages(component) else: generate_single_component_page(component) diff --git a/doc/getting_started/index.md b/doc/getting_started/index.md index dbc452a1..26152d5e 100644 --- a/doc/getting_started/index.md +++ b/doc/getting_started/index.md @@ -12,6 +12,13 @@ This _Getting Started_ guide will go through the installation of Lumen, how to b Install Lumen in a few easy steps ::: +:::: + +## AI + +::::{grid} 1 2 2 3 +:gutter: 1 1 1 2 + :::{grid-item-card} {octicon}`zap;2em;sd-mr-1` Using Lumen AI :link: using_lumen_ai :link-type: doc @@ -26,6 +33,13 @@ How to explore your data with Lumen AI Configure Lumen AI to use custom Coordinators, Agents, and LLM providers ::: +:::: + +## Core + +::::{grid} 1 2 2 3 +:gutter: 1 1 1 2 + :::{grid-item-card} {octicon}`tools;2em;sd-mr-1` Build a dashboard :link: build_dashboard :link-type: doc @@ -54,7 +68,8 @@ Get an overview of the core concepts of Lumen hidden: true --- installation -lumen_ai +using_lumen_ai +configuring_lumen_ai build_dashboard core_concepts pipelines diff --git a/doc/how_to/index.md b/doc/how_to/index.md index 23e01555..bb7765f5 100644 --- a/doc/how_to/index.md +++ b/doc/how_to/index.md @@ -43,7 +43,7 @@ Learn to create and use template prompts for influencing the AI's responses. ::: :::: -2 + ## Data Intake ::::{grid} 1 2 2 3 diff --git a/doc/index.md b/doc/index.md index 0c468e51..305d132b 100644 --- a/doc/index.md +++ b/doc/index.md @@ -26,7 +26,7 @@ Install Lumen in a few easy steps ::: :::{grid-item-card} {octicon}`zap;2em;sd-mr-1` Exploration with Lumen AI -:link: getting_started/lumen_ai +:link: getting_started/using_lumen_ai :link-type: doc How to explore your data with Lumen AI. diff --git a/lumen/ai/controls.py b/lumen/ai/controls.py index 59aa662c..e527b64e 100644 --- a/lumen/ai/controls.py +++ b/lumen/ai/controls.py @@ -4,7 +4,6 @@ import pandas as pd import param -from markitdown import MarkItDown from panel.layout import ( Column, FlexBox, Row, Tabs, ) @@ -123,9 +122,8 @@ class SourceControls(Viewer): def __init__(self, **params): super().__init__(**params) - - self._markitdown = MarkItDown() self.tables_tabs = Tabs(sizing_mode="stretch_width") + self._markitdown = None self._file_input = FileDropper( height=100, multiple=self.param.multiple, @@ -278,6 +276,9 @@ def _add_document( file: io.BytesIO, document_controls: DocumentControls ): + if self._markitdown is None: + from markitdown import MarkItDown + self._markitdown = MarkItDown() text = self._markitdown.convert_stream( file, file_extension=document_controls.extension ).text_content diff --git a/pixi.toml b/pixi.toml index 65b18fb6..7c8d2e00 100644 --- a/pixi.toml +++ b/pixi.toml @@ -1,6 +1,6 @@ [project] name = "lumen" -channels = ["pyviz/label/dev", "bokeh", "conda-forge"] +channels = ["pyviz/label/dev", "bokeh", "conda-forge", "conda-forge/label/markitdown_dev"] platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] [tasks] @@ -10,11 +10,11 @@ install = 'python -m pip install --no-deps --disable-pip-version-check -e .' PYTHONIOENCODING = "utf-8" [environments] -test-310 = ["py310", "test-core", "test"] -test-311 = ["py311", "test-core", "test"] -test-312 = ["py312", "test-core", "test"] +test-310 = ["py310", "test-core", "test", "ai", "sql"] +test-311 = ["py311", "test-core", "test", "ai", "sql"] +test-312 = ["py312", "test-core", "test", "ai", "sql"] test-core = ["py312", "test-core"] -docs = ["py311", "doc"] +docs = ["py311", "doc", "ai", "sql"] build = ["py311", "build"] lint = ["py311", "lint"] @@ -40,6 +40,28 @@ python = "3.11.*" [feature.py312.dependencies] python = "3.12.*" +[feature.ai.dependencies] +duckdb = "*" +griffe = "*" +instructor = ">=1.4.3" +markitdown = "*" +nbformat = "*" +openai = "*" +pyarrow = "*" +pydantic = ">=2.8.0" +pydantic-extra-types = "*" + +[feature.ai-local.dependencies] +huggingface_hub = "*" + +[feature.ai-llama.dependencies] +llama-cpp-python = ">=0.3.0" + +[feature.sql.dependencies] +duckdb = "*" +intake-sql = "*" +sqlalchemy = "*" + # ============================================= # =================== TESTS =================== # ============================================= @@ -57,44 +79,12 @@ dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests test-unit = 'pytest lumen/tests -n logical --dist loadgroup' [feature.test.dependencies] +dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests dask-core = "*" fastparquet = "*" matplotlib-base = ">=3.4" # Ubuntu + Python 3.9 installs old version matplotlib (3.3.2) msgpack-python = "*" toolz = "*" -dask = "<=2024.10" # Temporary workaround for pyarrow string issue in tests -# sql -intake-sql = "*" -python-duckdb = "*" -sqlalchemy = "*" -# ai -datashader = "*" -duckdb = "*" -griffe = "*" -instructor = ">=1.4.3" -nbformat = "*" -openai = "*" -pyarrow = "*" -pydantic = ">=2.8.0" - -[feature.test.pypi-dependencies] -pydantic_extra_types = "*" - -# [feature.ai.dependencies] -# datashader = "*" -# python-duckdb = "*" -# instructor = ">=1.4.3" -# nbformat = "*" -# openai = "*" -# pyarrow = "*" -# pydantic = ">=2.8.0" -# - -# [feature.ai-local.dependencies] -# huggingface_hub = "*" -# -# [feature.ai-llama.dependencies] -# llama-cpp-python = ">=0.3.0" # ============================================= # =================== DOCS ====================