Merge pull request #2 from oribarilan/ori/docs

running docs

README.md

@@ -97,10 +97,19 @@ Currently, there are 2 scenarios tested with varying dataset sizes (100, 10K, 1M
 * Scenario 1: zipping two iterables of Person objects, 
 and taking the first 5 (by age asc) that are of different gender.
 ```python
+from fliq import q
+from fliq.tests.fliq_test_utils import gen_people
+dataset = gen_people(100)
+shuffled = q(dataset).shuffle()
 q(dataset).zip(shuffled).where(lambda ps: ps[0].gender != ps[1].gender).order(by=lambda ps: ps[0].age+ps[1].age).take(5).to_list()
 ```
 * Scenario 2: filtering prepending and appending a list of Person objects 
 ```python
+from fliq import q
+from fliq.tests.fliq_test_utils import gen_people
+dataset = gen_people(100)
+first = gen_people(200)
+last = gen_people(200)
 q(dataset).where(lambda p: 0 <= p.age < 100).prepend_many(first).append_many(last).select(lambda p: p.name).to_list()
 ```
 

benchmark/benchmark.py

@@ -1,33 +1,13 @@
 import random
-from dataclasses import dataclass
 from typing import Iterable
 
 import pandas as pd
 import matplotlib.pyplot as plt
 
 from benchmark.benchmark_runner import BenchmarkRunner, NamedMethod
 from fliq import q
-
-@dataclass
-class Person:
-    name: str
-    age: int
-    gender: str
-
-
-def gen_name(num: int):
-    # list of names for every alphabet letter
-    names = ['anna', 'bob', 'charlie', 'david', 'emma', 'frank', 'gina', 'harry', 'ian', 'jane',
-             'kate', 'larry',
-             'mike', 'nancy', 'olivia', 'peter', 'quinn', 'robert', 'sarah', 'tom', 'ursula',
-             'victor', 'william',
-             'xavier', 'yvonne', 'zach']
-    return names[num % 26] + str(num)
-
-
-def gen_data(num: int):
-    return [Person(name=gen_name(x), age=x, gender='M' if x % 2 == 0 else 'F') for x in range(num)]
-
+from fliq.tests.fliq_test_utils import Person, gen_people
+
 
 # s1: get the oldest male
 
@@ -117,7 +97,7 @@ def format_ticks(x):
     scenario_name="Scenario 1",
     m1=NamedMethod("Fliq", s1_fliq),
     m2=NamedMethod("Standard Library", s1_std_lib),
-    dataset_generator=gen_data,
+    dataset_generator=gen_people,
 ).run(
     sizes=[
         1_00,
@@ -131,7 +111,7 @@ def format_ticks(x):
     scenario_name="Scenario 2",
     m1=NamedMethod("Fliq", s2_fliq),
     m2=NamedMethod("Standard Library", s2_std_lib),
-    dataset_generator=gen_data,
+    dataset_generator=gen_people,
 ).run(
     sizes=[
         1_00,

docs/index.md

@@ -97,10 +97,19 @@ Currently, there are 2 scenarios tested with varying dataset sizes (100, 10K, 1M
 * Scenario 1: zipping two iterables of Person objects, 
 and taking the first 5 (by age asc) that are of different gender.
 ```python
+from fliq import q
+from fliq.tests.fliq_test_utils import gen_people
+dataset = gen_people(100)
+shuffled = q(dataset).shuffle()
 q(dataset).zip(shuffled).where(lambda ps: ps[0].gender != ps[1].gender).order(by=lambda ps: ps[0].age+ps[1].age).take(5).to_list()
 ```
 * Scenario 2: filtering prepending and appending a list of Person objects 
 ```python
+from fliq import q
+from fliq.tests.fliq_test_utils import gen_people
+dataset = gen_people(100)
+first = gen_people(200)
+last = gen_people(200)
 q(dataset).where(lambda p: 0 <= p.age < 100).prepend_many(first).append_many(last).select(lambda p: p.name).to_list()
 ```
 

docs/reference/api_roadmap.md

@@ -1,12 +1,14 @@
 # API Roadmap
 
-## Special Functionality
+## V1 Functionality
+
+### Special Functionality
 
 - [x] [snap](snapshots.md) (aka cache, materialize)
 - [x] [partition](partitioning.md)
-- [x] peak
+- [x] [peek](peeking.md)
 
-## Mapper Methods
+### Mapper Methods
 
 - [x] [where](mapper_methods.md#fliq.query.Query.where) (aka filter)
 - [x] [select](mapper_methods.md#fliq.query.Query.select) (aka map)
@@ -30,21 +32,21 @@
 - [x] [shuffle](mapper_methods.md#fliq.query.Query.shuffle)
 - [x] [flatten](mapper_methods.md#fliq.query.Query.flatten)
 
-## Materializers
+### Materializers
 
 - [x] [contains](materializer_methods.md#fliq.query.Query.contains)
 - [x] [equals](materializer_methods.md#fliq.query.Query.equals)
 - [x] [to_list](materializer_methods.md#fliq.query.Query.to_list)
 - [x] [to_dict](materializer_methods.md#fliq.query.Query.to_dict)
 - [ ] conversion (to_set, to_tuple, to_string)
 
-### Special Materializers
+#### Special Materializers
 
 - [x] [in / not in](materializer_methods.md#fliq.query.Query.contains) (aka membership)
 - [x] [== / !=](materializer_methods.md#fliq.query.Query.__eq__) (aka equality)
 - [ ] arithmetic (+, -, *, , %)
 
-### Reducers
+#### Reducers
 
 - [x] [first](materializer_methods.md#fliq.query.Query.first)
 - [x] [first_or_default](materializer_methods.md#fliq.query.Query.first_or_default)
@@ -54,8 +56,17 @@
 - [x] [all](materializer_methods.md#fliq.query.Query.all)
 - [x] [aggregate](materializer_methods.md#fliq.query.Query.aggregate)
 
-#### Numeric Reducers
+##### Numeric Reducers
 - [x] [sum](materializer_methods.md#fliq.query.Query.sum)
 - [x] [min](materializer_methods.md#fliq.query.Query.min)
 - [x] [max](materializer_methods.md#fliq.query.Query.max)
-- [ ] average
+- [ ] average
+
+## V2 Functionality
+
+V1 is still going strong, add lots of functionality.
+
+However, we can already say that future versions will include async functionality:
+
+* V2 will include async manipulation over iterables.
+* V3 will include async manipulation over async iterables.

fliq/tests/fliq_test_utils.py

@@ -3,6 +3,34 @@
 from typing import Optional
 
 
+@dataclass
+class Person:
+    name: str
+    age: int
+    gender: str
+
+
+def gen_person_name(num: int):
+    # list of names for every alphabet letter
+    names = ['anna', 'bob', 'charlie', 'david', 'emma', 'frank', 'gina', 'harry', 'ian', 'jane',
+             'kate', 'larry',
+             'mike', 'nancy', 'olivia', 'peter', 'quinn', 'robert', 'sarah', 'tom', 'ursula',
+             'victor', 'william',
+             'xavier', 'yvonne', 'zach']
+    return names[num % 26] + str(num)
+
+
+def gen_people(num: int):
+    return [
+        Person(
+            name=gen_person_name(x),
+            age=x,
+            gender='M' if x % 2 == 0 else 'F'
+        )
+        for x in range(num)
+    ]
+
+
 @dataclass
 class Point:
     x: int

fliq/tests/system/test_sanity.py

@@ -2,6 +2,8 @@
 from unittest import TestCase
 
 from fliq import q
+from fliq.tests.fliq_test_utils import Person
+
 
 @dataclass
 class Item:
@@ -14,11 +16,21 @@ class TestSanity(TestCase):
     def setUp(self):
         self.items = (Item(i, i % 5) for i in range(100))
 
-    def test_sanity(self):
+    def test_sanity_1(self):
         price = (
             q(self.items)
             .where(lambda x: x.price > 50)
             .select(lambda x: x.price)
             .first_or_default(default=0)
         )
         assert price == 51
+
+    def test_sanity_2(self):
+        men = [Person(name=f'Mr {i}', age=i, gender="M") for i in range(10)]
+        women = [Person(name=f'Ms {i+10}', age=i+10, gender="W") for i in range(10)]
+        non_binary = [Person(name=f'{i+30}', age=i+30, gender="N") for i in range(10)]
+        (q(men)
+         .append_many(women)
+         .where(lambda p: p.age < 5)
+         .prepend_many(non_binary)
+         .select(lambda p: p.name).to_list())

justfile

@@ -38,6 +38,7 @@ check:
 doc:
     python -m doctest fliq/query.py
     python scripts/gen_docs.py
+    python scripts/execute_doc_files.py
     mkdocs build
 
 # test, build and run docs server

scripts/execute_doc_files.py

@@ -0,0 +1,58 @@
+import ast
+import re
+from pathlib import Path
+from typing import List
+
+
+class MarkdownPythonEvaluator:
+    def __init__(self, markdown_file_path: Path):
+        self.markdown_file_path = markdown_file_path
+        self.code_block_pattern = re.compile(r'```python(.*?)```', re.DOTALL)
+
+    def extract_python_code_blocks(self) -> List[str]:
+        """Extract python code blocks from the markdown file and split them into statements."""
+        code_blocks = []
+        with open(self.markdown_file_path, 'r', encoding='utf-8') as md_file:
+            content = md_file.read()
+
+            # Pattern to match python code blocks
+            pattern = re.compile(r'```python(.*?)```', re.DOTALL)
+
+            # Iterate over all matches
+            for match in pattern.finditer(content):
+                code_block = match.group(1).strip()  # Extract the code block
+                code_blocks.append(code_block)
+
+        return code_blocks
+
+    def eval(self, code: str):
+        """
+        Safely evaluate a single python code block.
+        This method does not allow modification of the environment or execution of arbitrary code.
+        """
+        try:
+            # Safely evaluate expressions, preventing code execution
+            block = ast.parse(code, mode='exec')
+            exec(compile(block, '<string>', mode='exec'))
+        except Exception as e:
+            # In real use, you might want to handle the exception differently
+            print(f"Error evaluating code block at {self.markdown_file_path}: "
+                  f"{e} at block:\n{code}")
+            exit(1)
+
+    def evaluate_blocks(self):
+        """Evaluate all extracted Python code blocks."""
+        code_blocks = self.extract_python_code_blocks()
+        for block in code_blocks:
+            self.eval(block)
+
+
+# Usage
+root_path = Path(__file__).parent.parent
+readme_path = root_path / 'README.md'
+docs_path = root_path / 'docs'
+files = [readme_path] + list(docs_path.glob('**/*.md'))
+for f in files:
+    print(f"Executing code blocks in {f}")
+    evaluator = MarkdownPythonEvaluator(f)
+    evaluator.evaluate_blocks()