From d79d21cde3415424a4a43a6d673251d91548644f Mon Sep 17 00:00:00 2001 From: Chris Clark Date: Tue, 10 Sep 2024 09:20:55 -0400 Subject: [PATCH] 5.3b2 (#670) 5.3b2 --- HISTORY.rst | 4 +- explorer/__init__.py | 2 +- explorer/app_settings.py | 2 + explorer/ee/db_connections/models.py | 1 - explorer/ee/db_connections/views.py | 1 - explorer/forms.py | 7 +- explorer/permissions.py | 4 +- explorer/src/js/assistant.js | 21 +- .../assistant/table_description_list.html | 5 +- .../connections/connection_upload.html | 13 +- .../templates/connections/connections.html | 11 +- explorer/templates/explorer/assistant.html | 5 +- explorer/templates/explorer/base.html | 6 + explorer/templates/explorer/play.html | 3 +- explorer/templates/explorer/query.html | 7 +- explorer/templates/explorer/query_list.html | 310 ++++++++++-------- explorer/views/list.py | 2 + explorer/views/mixins.py | 1 + 18 files changed, 239 insertions(+), 166 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ac35f278..2241f230 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,7 @@ Change Log This document records all notable changes to `SQL Explorer `_. This project adheres to `Semantic Versioning `_. -`5.3.0 (beta)`_ (2024-08-29) +`5.3.0 (beta 2)`_ (2024-09-10) =========================== * `#664`_: Improvements to the AI SQL Assistant: @@ -41,6 +41,8 @@ This project adheres to `Semantic Versioning `_. * Fixed a bug when validating connections to uploaded files. Also added basic locking when downloading files from S3. +* On-boarding UI; if no connections or queries are created, the UI walks the user through it a bit. + * Keyboard shortcut for formatting the SQL in the editor. - Cmd+Shift+F (Windows: Ctrl+Shift+F) diff --git a/explorer/__init__.py b/explorer/__init__.py index f0e2859b..ad2ee089 100644 --- a/explorer/__init__.py +++ b/explorer/__init__.py @@ -3,7 +3,7 @@ "minor": 3, "patch": 0, "releaselevel": "beta", - "serial": 1 + "serial": 2 } diff --git a/explorer/app_settings.py b/explorer/app_settings.py index 8bf69b63..a25a336e 100644 --- a/explorer/app_settings.py +++ b/explorer/app_settings.py @@ -170,6 +170,8 @@ # 500mb default max EXPLORER_MAX_UPLOAD_SIZE = getattr(settings, "EXPLORER_MAX_UPLOAD_SIZE", 500 * 1024 * 1024) +EXPLORER_HOSTED = getattr(settings, "EXPLORER_HOSTED", False) + def has_assistant(): return EXPLORER_AI_API_KEY is not None diff --git a/explorer/ee/db_connections/models.py b/explorer/ee/db_connections/models.py index 140543ac..49d853a1 100644 --- a/explorer/ee/db_connections/models.py +++ b/explorer/ee/db_connections/models.py @@ -151,7 +151,6 @@ def save(self, *args, **kwargs): DatabaseConnection.objects.filter(default=True).update(default=False) else: # If there is no default set yet, make this newly created one the default. - # Effectively this is for first-time installations. has_default = DatabaseConnection.objects.filter(default=True).exists() if not has_default: self.default = True diff --git a/explorer/ee/db_connections/views.py b/explorer/ee/db_connections/views.py index 3f5c904e..0bef17ae 100644 --- a/explorer/ee/db_connections/views.py +++ b/explorer/ee/db_connections/views.py @@ -82,7 +82,6 @@ def post(self, request): # noqa class DatabaseConnectionsListView(PermissionRequiredMixin, ExplorerContextMixin, ListView): - context_object_name = "sqlite_uploads" permission_required = "connections_permission" template_name = "connections/connections.html" model = DatabaseConnection diff --git a/explorer/forms.py b/explorer/forms.py index e025b558..a28b1db0 100644 --- a/explorer/forms.py +++ b/explorer/forms.py @@ -41,7 +41,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["database_connection"].widget.choices = self.connections if not self.instance.database_connection: - self.initial["database_connection"] = default_db_connection().alias + default_db = default_db_connection() + self.initial["database_connection"] = default_db_connection().alias if default_db else None self.fields["database_connection"].widget.attrs["class"] = "form-select" def clean(self): @@ -66,6 +67,10 @@ def created_at_time(self): @property def connections(self): + default_db = default_db_connection() + if default_db is None: + return [] + # Ensure the default connection appears first in the dropdown in the form result = DatabaseConnection.objects.annotate( custom_order=Case( diff --git a/explorer/permissions.py b/explorer/permissions.py index 8704decf..02417a7e 100644 --- a/explorer/permissions.py +++ b/explorer/permissions.py @@ -16,7 +16,7 @@ def view_permission(request, **kwargs): # And token auth does not give you permission to view the list. -def view_permission_list(request): +def view_permission_list(request, *args, **kwargs): return app_settings.EXPLORER_PERMISSION_VIEW(request)\ or allowed_query_pks(request.user.id) @@ -25,5 +25,5 @@ def change_permission(request, *args, **kwargs): return app_settings.EXPLORER_PERMISSION_CHANGE(request) -def connections_permission(request, **kwargs): +def connections_permission(request, *args, **kwargs): return app_settings.EXPLORER_PERMISSION_CONNECTIONS(request) diff --git a/explorer/src/js/assistant.js b/explorer/src/js/assistant.js index 8e7ce159..17d576eb 100644 --- a/explorer/src/js/assistant.js +++ b/explorer/src/js/assistant.js @@ -46,7 +46,8 @@ function setupTableList() { removeItemButton: true, searchEnabled: true, shouldSort: false, - position: 'bottom' + position: 'bottom', + placeholderValue: "Click to search for relevant tables" }); // TODO - nasty. Should be refactored. Used by submitAssistantAsk to get relevant tables. @@ -66,6 +67,16 @@ function setupTableList() { }); }); + const refreshTables = document.getElementById('refresh_tables_button'); + refreshTables.addEventListener('click', (e) => { + e.preventDefault(); + keys.forEach(k => { + choices.removeActiveItemsByValue(k); + }); + selectRelevantTablesSql(choices, keys) + selectRelevantTablesRequest(choices, keys) + }); + selectRelevantTablesSql(choices, keys); document.addEventListener('docChanged', debounce( @@ -81,16 +92,16 @@ function setupTableList() { } function selectRelevantTablesSql(choices, keys) { - const textContent = window.editor.state.doc.toString(); + const textContent = window.editor.state.doc.toString().toLowerCase(); const textWords = new Set(textContent.split(/\s+/)); - const hasKeys = keys.filter(key => textWords.has(key)); + const hasKeys = keys.filter(key => textWords.has(key.toLowerCase())); choices.setChoiceByValue(hasKeys); } function selectRelevantTablesRequest(choices, keys) { const textContent = document.getElementById("id_assistant_input").value - const textWords = new Set(textContent.split(/\s+/)); - const hasKeys = keys.filter(key => textWords.has(key)); + const textWords = new Set(textContent.toLowerCase().split(/\s+/)); + const hasKeys = keys.filter(key => textWords.has(key.toLowerCase())); choices.setChoiceByValue(hasKeys); } diff --git a/explorer/templates/assistant/table_description_list.html b/explorer/templates/assistant/table_description_list.html index 1ae59f3d..56410c1e 100644 --- a/explorer/templates/assistant/table_description_list.html +++ b/explorer/templates/assistant/table_description_list.html @@ -3,9 +3,10 @@ {% block sql_explorer_content %}

Table Annotations

-

These will be injected into any AI assistant prompts that reference the annotated table.

+

Write some notes about your tables to help the AI Assistant do its job. Relevant annotations will be automatically injected into AI assistant requests.

+

Good annotations may describe the purposes of columns that are not obvious from their name alone, common joins to other tables, or the semantic meaning of enum values.

- Create New + Create Annotation diff --git a/explorer/templates/connections/connection_upload.html b/explorer/templates/connections/connection_upload.html index 4f79d1ff..1124f43d 100644 --- a/explorer/templates/connections/connection_upload.html +++ b/explorer/templates/connections/connection_upload.html @@ -2,24 +2,21 @@ {% block sql_explorer_content %}
-

Upload a file

+

Create a connection from an uploaded file

Supports .csv, .json, .db, and .sqlite files. JSON files with one JSON document per line are also supported. CSV/JSON data will be parsed and converted to SQLite. SQLite databases must not be password protected.

-

- +

Appending to an existing connection will add a new table to the SQLite database, named after the uploaded file. If a table with the filename already exists, it will be replaced with the uploaded data.

- - When appending, if a table with the filename already exists, it will be replaced with the uploaded data. +
-
-

Drag and drop, or click to upload .csv, .json, .db, .sqlite.

+

Upload: Drag and drop, or click to upload .csv, .json, .db, .sqlite.

0%
diff --git a/explorer/templates/connections/connections.html b/explorer/templates/connections/connections.html index beaffc73..48cf6b24 100644 --- a/explorer/templates/connections/connections.html +++ b/explorer/templates/connections/connections.html @@ -5,8 +5,15 @@

Connections

- Add New Connection - Upload File +
+ Add New Connection + Upload File + {% if object_list|length == 0 %} + + Connect to an existing database, or upload a csv, json, or sqlite file. + + {% endif %} +
diff --git a/explorer/templates/explorer/assistant.html b/explorer/templates/explorer/assistant.html index afec2aa0..4e5bf18a 100644 --- a/explorer/templates/explorer/assistant.html +++ b/explorer/templates/explorer/assistant.html @@ -33,11 +33,14 @@
- +
+
+ +
diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 32bac5ba..1b4023c4 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -80,6 +80,12 @@

This is easy to fix, I promise!

{% translate "Favorites" %} + {% if hosted %} +
  • + Manage +
    • + {% endif %}
    diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index eb3935c0..ded60a33 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -16,7 +16,7 @@

    {% translate "Playground" %}

    {{ error|escape }}
    {% endif %} {{ form.non_field_errors }} - {% if form.connections|length > 1 and can_change %} + {% if can_change %}
    {{ form.database_connection }} @@ -26,7 +26,6 @@

    {% translate "Playground" %}

    {{ form.database_connection }}
    -
    {{ form.database_connection.value }}
    {% endif %}
    diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index ce640aba..e07127d9 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -39,7 +39,7 @@

    {{ error|escape }}
    {% endfor %}{% endif %}

    - {% if form.connections|length > 1 and can_change %} + {% if can_change %}
    {{ form.database_connection }} @@ -96,6 +96,10 @@

    {% if query %}
    + + {% if query and can_change and assistant_enabled %}{{ form.few_shot }} {% translate "Assistant Example" %}{% endif %} + + @@ -150,6 +154,5 @@

    {% if query and can_change and tasks_enabled %}{{ form.snapshot }} {% translate "Snapshot" %}{% endif %} - {% if query and can_change and assistant_enabled %}{{ form.few_shot }} {% translate "Assistant Example" %}{% endif %}
    {% endblock %} diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index 9bd321ca..a8d5f607 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -3,160 +3,196 @@ {% block sql_explorer_content %}
    {% csrf_token %}
    - {% if recent_queries|length > 0 %} -
    -

    {% translate "Recently Run by You" %}

    + {% if connection_count == 0 %} +
    +
    +
    +

    Welcome to SQL Explorer!

    +

    + First things first, in order to create queries and start exploring, you'll need to: +

    +

    + Create a Connection +

    +

    + Need help? Check out the documentation{% if hosted %} or contact support{% endif %}. +

    +
    +
    +
    + {% elif object_list|length == 0 %} +
    +
    +
    +

    Time to create a query

    +

    + You have {{ connection_count }} connection{% if connection_count > 1 %}s{% endif %} created, now get cracking! +

    +

    + Create a Query or Play Around +

    +

    + Need help? Check out the documentation{% if hosted %} or contact support{% endif %}. +

    +
    +
    +
    + {% else %} + {% if recent_queries|length > 0 %} +
    +

    {% translate "Recently Run by You" %}

    +

    + + + + + + + + + {% for object in recent_queries %} + + + + + + {% endfor %} + +
    {% translate "Query" %}{% translate "Last Run" %}CSV
    + {{ object.query.title }} + {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }} + + + +
    +
    + {% endif %} + +
    +
    +
    +

    {% translate "All Queries" %}

    +
    +
    + +
    +
    - - - + + + {% if tasks_enabled %} + + {% endif %} + + {% if can_change %} + + + {% endif %} + + + + - - {% for object in recent_queries %} - - - - - - {% endfor %} - -
    {% translate "Query" %}{% translate "Last Run" %}CSV{% translate "Query" %}{% translate "Created" %}{% translate "Email" %}{% translate "CSV" %}{% translate "Play" %}{% translate "Delete" %}{% translate "Favorite" %}{% translate "Last Run" %}{% translate "Run Count" %}{% translate "Connection" %}
    - {{ object.query.title }} - {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }} - - - -
    -
    - {% endif %} - -
    -
    -
    -

    {% translate "All Queries" %}

    -
    -
    - -
    -
    - - - - - - {% if tasks_enabled %} - - {% endif %} - - {% if can_change %} - - - {% endif %} - - - - - - - - {% for object in object_list %} - - {% if object.is_header %} - - {% else %} - - + {% for object in object_list %} + + {% if object.is_header %} + + {% else %} + + + {% if tasks_enabled %} + {% endif %} - - {% if tasks_enabled %} - - {% endif %} - - {% if can_change %} - + + {% endif %} + + + + {% endif %} - - - - - {% endif %} - - {% endfor %} - -
    {% translate "Query" %}{% translate "Created" %}{% translate "Email" %}{% translate "CSV" %}{% translate "Play" %}{% translate "Delete" %}{% translate "Favorite" %}{% translate "Last Run" %}{% translate "Run Count" %}{% translate "Connection" %}
    - - - {{ object.title }} ({{ object.count }}) - - - - {{ object.title }} - {{ object.created_at|date:"m/d/y" }} - {% if object.created_by_user %} - {% blocktranslate trimmed with cuser=object.created_by_user %} - by {{cuser}} - {% endblocktranslate %} +
    + + + {{ object.title }} ({{ object.count }}) + + + + {{ object.title }} + {{ object.created_at|date:"m/d/y" }} + {% if object.created_by_user %} + {% blocktranslate trimmed with cuser=object.created_by_user %} + by {{cuser}} + {% endblocktranslate %} + {% endif %} + + + - - - - - - - - + + - - - + {% if can_change %} + + + + + + + + + {% query_favorite_button object.id object.is_favorite 'query_favorite_toggle' %}{% if object.ran_successfully %} + + {% elif object.ran_successfully is not None %} + + {% endif %} + {{ object.last_run_at|date:"m/d/y" }} {{ object.run_count }}{{ object.connection_name }} {% query_favorite_button object.id object.is_favorite 'query_favorite_toggle' %}{% if object.ran_successfully %} - - {% elif object.ran_successfully is not None %} - - {% endif %} - {{ object.last_run_at|date:"m/d/y" }} - {{ object.run_count }}{{ object.connection_name }}
    -
    + + {% endfor %} + + +
    -