Merge pull request #150 from NLP4ALL/feature/update-hierarchy

Feature/update hierarchy

app.dev.env

@@ -1,8 +1,11 @@
 # environment variables for docker (app specific)
 DB_BACKEND=postgres
 POSTGRES_HOST=db-dev
+MONGODB_HOST=document-store-dev
 NLP4ALL_ENV=development
-DEBUG_FLASK_APP=0
+DEBUG_FLASK_APP="0"
+FLASK_AUTO_RELOAD="1"
 NLP4ALL_ADMIN_EMAIL=example@example.org
 NLP4ALL_ADMIN_PASSWORD=example
-NLP4ALL_ORG_NAME=Example
+NLP4ALL_ORG_NAME=Example
+MQ_HOST=rabbitmq-dev

compose.development.yaml

@@ -1,24 +1,33 @@
 services:
  db-dev:
  restart: always
- image: postgres:15.1
+ image: postgres:15.2
  volumes:
  - postgres_data_dev:/var/lib/postgresql/data/
- expose:
- - "5432"
+ ports:
+ - "5432:5432"
+ networks:
+ - back-tier
+ env_file:
+ - ./db.dev.env
+ document-store-dev:
+ restart: always
+ image: mongo:5.0.15
+ ports:
+ - "27017:27017"
  networks:
  - back-tier
+ volumes:
+ - mongo_data_dev:/data/db
  env_file:
  - ./db.dev.env
- rabbitmq:
+ rabbitmq-dev:
  restart: always
  image: rabbitmq:3.11
  expose:
  - "5672"
  networks:
  - back-tier
- volumes:
- - "rabbit:/var/lib/rabbitmq"
  app-dev:
  restart: always
  build: 
@@ -39,15 +48,15 @@ services:
  - back-tier
  depends_on:
  - db-dev
- - rabbitmq
  - worker-dev
+ - document-store-dev
  worker-dev:
  restart: always
  build: 
  context: .
  target: nlpapp-dev
  dockerfile: ./docker/python/Dockerfile
- command: celery -A make_celery worker -l info
+ command: bash -c "watchmedo auto-restart --recursive --pattern='*.py' -- celery -A make_celery worker -l info"
  volumes:
  - ./nlp4all:/home/app/nlp4all
  env_file:
@@ -57,9 +66,11 @@ services:
  - back-tier
  depends_on:
  - db-dev
- - rabbitmq
+ - rabbitmq-dev
+ - document-store-dev
 networks:
  front-tier: {}
  back-tier: {}
 volumes:
  postgres_data_dev:
+ mongo_data_dev:

compose.yaml

@@ -17,8 +17,17 @@ services:
  - "5672"
  networks:
  - back-tier
+ document-store:
+ restart: always
+ image: mongo:5.0.15
+ ports:
+ - "27017:27017"
+ networks:
+ - back-tier
  volumes:
- - "rabbit:/var/lib/rabbitmq"
+ - mongo_data:/data/db
+ env_file:
+ - ./db.env
  app:
  restart: always
  build: 
@@ -40,6 +49,7 @@ services:
  - nlpdb
  - rabbitmq
  - worker
+ - document-store
  worker:
  restart: always
  build: 
@@ -57,6 +67,7 @@ services:
  depends_on:
  - nlpdb
  - rabbitmq
+ - document-store
  web:
  restart: always
  build: 
@@ -74,4 +85,5 @@ networks:
  back-tier: {}
 volumes:
  postgres_data_prod:
- static_volume:
+ static_volume:
+ mongo_data:

db.dev.env

@@ -2,4 +2,6 @@
 POSTGRES_USER=nlp4all
 POSTGRES_PASSWORD=nlp4all
 POSTGRES_DB=nlp4all
-POSTGRES_PORT=5432
+POSTGRES_PORT=5432
+MONGO_INITDB_ROOT_USERNAME=nlp4all
+MONGO_INITDB_ROOT_PASSWORD=nlp4all

db.env-example

@@ -2,4 +2,6 @@
 POSTGRES_USER=nlp4all
 POSTGRES_PASSWORD=nlp4all
 POSTGRES_DB=nlp4all
-POSTGRES_PORT=5432
+POSTGRES_PORT=5432
+MONGO_INITDB_ROOT_USERNAME=nlp4all
+MONGO_INITDB_ROOT_PASSWORD=nlp4all

docker/nginx/Dockerfile

@@ -1,3 +1,3 @@
-FROM nginx:1.23.2
+FROM nginx:1.23.4
 RUN rm /etc/nginx/conf.d/default.conf
 COPY ./nginx.conf /etc/nginx/nginx.conf

docker/python/Dockerfile

@@ -73,10 +73,14 @@ FROM base as nlpapp-dev
 
 # add dev requirements
 COPY --from=builder /usr/src/app/requirements-dev.txt .
+RUN apt-get install -y libyaml-dev inotify-tools
 RUN pip install --progress-bar off --no-cache-dir -r requirements-dev.txt
 # install debugpy for remote debug listening
 RUN pip install debugpy
 
+# temporary fix until werkzeug > 2.2.3 is released
+RUN pip install https://github.com/pallets/werkzeug/archive/refs/heads/main.zip
+
 COPY ./docker/python/run-nlp4all.sh $HOME
 RUN chmod +x $HOME/run-nlp4all.sh
 RUN chown app:app $HOME/run-nlp4all.sh

docker/python/run-nlp4all.sh

@@ -1,11 +1,15 @@
 #!/bin/sh
 
 # Check if DEBUG_FLASK_APP = 1
+RELOAD_CMD="--no-reload"
+if [ "$FLASK_AUTO_RELOAD" = "1" ]; then
+ RELOAD_CMD=""
+fi
 if [ "$DEBUG_FLASK_APP" = "1" ]; then
  echo "Running NLP4ALL for VSCode Debugging"
- python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m flask --debug --app nlp4all run --host=0.0.0.0 --port=5000
+ python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m flask --no-debugger --app nlp4all run --no-reload --host=0.0.0.0 --port=5000
 else
  echo "Running NLP4ALL Development Server"
- python -m flask --debug --app nlp4all run --host=0.0.0.0 --port=5000
+ python -m flask --debug --app nlp4all run $RELOAD_CMD --host=0.0.0.0 --port=5000
 fi
 

nlp4all/__init__.py

@@ -14,6 +14,7 @@
 
 from .helpers import database as dbhelper
 from .helpers import nlp
+from .helpers.mongo import Mongo
 from .helpers.celery import celery_init_app
 from .config import get_config, Config
 from .database import Base, nlp_sa_meta
@@ -24,15 +25,18 @@
  metadata=nlp_sa_meta,
  model_class=Base,
  engine_options={"future": True})
+docdb: Mongo = Mongo()
 migrate = Migrate()
 csrf = CSRFProtect()
 
+conf: Config = Config()
+
 
 def create_app(env: Union[None, str] = None) -> Flask:
  """Create the Flask app."""
-
+ global conf # pylint: disable=global-statement
  app = Flask(__name__, template_folder="views", static_folder=None)
- conf: Config = get_config(env)
+ conf = get_config(env, app)
  app.config.from_object(conf)
 
  db.init_app(app)
@@ -49,19 +53,22 @@ def create_app(env: Union[None, str] = None) -> Flask:
  login_manager = LoginManager(app)
  login_manager.user_loader(load_user)
  login_manager.login_message_category = "info"
- login_manager.refresh_view = "user_controller.reauth"
+ login_manager.refresh_view = "user_controller.reauth" # type: ignore
  login_manager.needs_refresh_message = (
  u"To protect your account, please reauthenticate to access this page."
  )
  login_manager.needs_refresh_message_category = "info"
 
  dbhelper.init_app(app)
+ # TODO: the connection probably doesn't need to exist for each request
+ docdb.init_app(app)
  nlp.init_app(app)
 
  # in non-production environments, we want to be able to get a list of routes
  if conf.env != "production":
  from .helpers import development # pylint: disable=import-outside-toplevel
  app.add_url_rule('/api/help', methods=['GET'], view_func=development.help_route)
+ app.add_url_rule('/api/get_ds/<int:dsid>', methods=['GET'], view_func=development.load_data_source)
 
  if conf.DB_BACKEND == "sqlite":
  from .helpers.database import model_cols_jsonb_to_json # pylint: disable=import-outside-toplevel

nlp4all/config.py

@@ -1,11 +1,15 @@
 """Flask configuration"""
 
+import typing as t
 import os
 import logging
 import secrets
 from pathlib import Path
 # from dotenv import load_dotenv
 
+if t.TYPE_CHECKING:
+ from flask import Flask
+
 
 def get_env_variable(name: str) -> str:
  """Get the environment variable or raise exception."""
@@ -28,6 +32,15 @@ def get_env_variable(name: str) -> str:
 except EnvironmentError:
  DB_BACKEND = "sqlite"
 
+try:
+ MONGODB_HOST = get_env_variable("MONGODB_HOST")
+ MONGO_INITDB_ROOT_USERNAME = get_env_variable("MONGO_INITDB_ROOT_USERNAME")
+ MONGO_INITDB_ROOT_PASSWORD = get_env_variable("MONGO_INITDB_ROOT_PASSWORD")
+except EnvironmentError:
+ MONGODB_HOST = "document-store"
+ MONGO_INITDB_ROOT_USERNAME = "nlp4all"
+ MONGO_INITDB_ROOT_PASSWORD = "nlp4all"
+
 if DB_BACKEND == "postgres":
  pg_host = get_env_variable('POSTGRES_HOST')
  pg_user = get_env_variable('POSTGRES_USER')
@@ -53,6 +66,7 @@ class Config: # pylint: disable=too-few-public-methods
  SECRET_FILE_PATH = Path(".flask_secret")
  DB_BACKEND = DB_BACKEND
  LOG_LEVEL = logging.WARNING
+ DATA_UPLOAD_DIR: str = 'data'
 
  # Security
  BCRYPT_LOG_ROUNDS: int = 12
@@ -139,29 +153,34 @@ class ProductionConfig(Config): # pylint: disable=too-few-public-methods
  """Configuration for the Flask app in production."""
 
 
-def get_config(env=None):
+def get_config(env=None, app: t.Optional['Flask'] = None):
  """Get the configuration for the Flask app."""
  if env is None:
  try:
  env = get_env_variable('NLP4ALL_ENV')
  except EnvironmentError:
  env = 'production'
  print(f'env is not set, using: {env}')
-
+ conf: t.Union[Config, None] = None
  if env == 'production':
  if DB_URI == "":
  raise EnvironmentError('Cannot use SQLite in production')
- return ProductionConfig(env)
+ conf = ProductionConfig(env)
 
  if env == 'testing':
- return TestConfig(env)
+ conf = TestConfig(env)
 
  if env == 'development':
  if DB_URI == "":
  raise EnvironmentError('Cannot use SQLite in production')
- return DevelopmentConfig(env)
+ conf = DevelopmentConfig(env)
 
  if env == 'localdev':
- return LocalDevelopmentConfig(env)
+ conf = LocalDevelopmentConfig(env)
+
+ if conf is not None:
+ if app is not None:
+ conf.DATA_UPLOAD_DIR = os.path.join(app.root_path, conf.DATA_UPLOAD_DIR)
+ return conf
 
  raise EnvironmentError(f'Unknown environment: {env}')

nlp4all/controllers/admin.py

@@ -3,7 +3,7 @@
 from flask import flash, redirect, url_for
 from celery.result import AsyncResult
 from nlp4all import db
-from ..models import OrganizationModel, DataTagCategoryModel
+from ..models import UserGroupModel, DataTagCategoryModel
 from ..forms.admin import AddOrgForm
 from ..forms.analyses import AddTweetCategoryForm
 from ..helpers.tweets import add_tweets_from_account
@@ -32,18 +32,18 @@ def manage_categories(cls):
  def add_org(cls):
  """Add organization page"""
  form = AddOrgForm()
- orgs = OrganizationModel.query.all()
+ orgs = UserGroupModel.query.all()
  if form.validate_on_submit():
- org = OrganizationModel(name=form.name.data)
+ org = UserGroupModel(name=form.name.data)
  db.add(org)
  db.session.commit()
- flash("Your organization has been created!", "success")
+ flash("Your group has been created!", "success")
  return redirect(url_for("admin_controller.add_org"))
  return cls.render_template("add_org.html", form=form, orgs=orgs)
 
  @classmethod
  def celery_test(cls, x: int, y: int):
- result = add_test.delay(x, y)
+ result = add_test.delay(x, y) # type: ignore
  return redirect(url_for("admin_controller.celery_result", task_id=result.id))
 
  @classmethod