Skip to content

Commit

Permalink
Reapplied last fixes before transition to public repo
Browse files Browse the repository at this point in the history
  • Loading branch information
kamudadreieck committed Mar 15, 2024
1 parent fe616b3 commit c6033cd
Show file tree
Hide file tree
Showing 23 changed files with 351 additions and 38 deletions.
16 changes: 16 additions & 0 deletions pipeline/core/migrations/0015_tenant_audescribe_active.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('core', '0014_auto_20230213_0505'),
]

operations = [
migrations.AddField(
model_name='tenant',
name='audescribe_active',
field=models.BooleanField(default=False),
),
]
4 changes: 4 additions & 0 deletions pipeline/core/models/tenant.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class Tenant(models.Model):
aws_active = models.BooleanField(default=True)
mllp_active = models.BooleanField(default=True)
deepl_active = models.BooleanField(default=False)
audescribe_active = models.BooleanField(default=False)

def get_secret(self, key, raise_exception=True, default=None):
if key in self.secrets:
Expand Down Expand Up @@ -57,6 +58,9 @@ def active_cloud_services(self):
if self.deepl_active:
l.append(Subtitle.Origin.DEEPL)

if self.audescribe_active:
l.append(Subtitle.Origin.AUDESCRIBE)

l.append(TranslationService.MANUAL)

return l
Expand Down
Empty file.
2 changes: 1 addition & 1 deletion pipeline/mooclink/services/aws_translation_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def fetch_translation_jobs(self, video_id):
if len(video.workflow_data['waiting_job_ids']) == 0:
video.workflow_status = None
if periodic_task_id := video.workflow_data.get('periodic_task_id'):
PeriodicTask.objects.get(pk=periodic_task_id).delete()
PeriodicTask.objects.filter(pk=periodic_task_id).delete()
video.workflow_data['cleared'] = True

video.save()
Expand Down
30 changes: 30 additions & 0 deletions pipeline/mooclink/services/periodic_task_service.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import json
from datetime import datetime, timedelta

from django_celery_beat.models import IntervalSchedule, PeriodicTask


class PeriodicTaskService:

@classmethod
def create_periodic_task(cls, service_type, task, video, start_time=None, end_time=None):
schedule, created = IntervalSchedule.objects.get_or_create(
every=10,
period=IntervalSchedule.MINUTES,
)

if PeriodicTask.objects.filter(name=f"Check AWS StandaloneTranslation for video={video.pk}").count():
return

periodic = PeriodicTask.objects.create(
interval=schedule,
name=f'Check AWS StandaloneTranslation for video={video.pk}',
task=task,
start_time=datetime.now() + timedelta(minutes=15),
kwargs=json.dumps({
'tenant_id': video.tenant_id,
'video_id': video.pk,
})
)

return periodic
72 changes: 72 additions & 0 deletions pipeline/mooclink/templates/mooclink/admin/job_view.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
{% extends "subtitles/base.html" %}
{% block content %}
{% load static %}
{% load utils %}
<link rel="stylesheet" href="{% static 'subtitles/css/mooclink.css' %}">

<div class="container-fluid">
<h1>Pending Videos</h1>

<table class="table table-sm">
<thead>
<tr>
<th>Video</th>
<th>Service</th>
<th>Started</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for video in pending_videos %}
<tr>
<td>
<a href="{% tenant_url 'mooclink.video.index' course_id=video.course_section.course.ext_id video_id=video.ext_id tenant=video.tenant %}" target="_blank">
{{ video }}
</a>
</td>
<td>{{ video.workflow_status }}</td>
<td>{{ video.job_initiated }}</td>
<td>
<button form="cancellation-form" type="submit" name="video_id" value="{{ video.id }}"
class="btn btn-danger btn-sm"
onclick="confirm('Please confirm the cancellation of job for video `{{ video }}`')">
Cancel
</button>
</td>
</tr>
{% endfor %}
</tbody>
</table>

<h2>Orphaned Periodic Tasks</h2>
<table class="table table-sm">
<thead>
<tr>
<th>Video</th>
<th>Started</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for orphaned in orphaned_tasks %}
<tr>
<td>
<a href="{% tenant_url 'mooclink.video.index' course_id=orphaned.video.course_section.course.ext_id video_id=orphaned.video.ext_id tenant=orphaned.video.tenant %}" target="_blank">
{{ orphaned.video }}
</a>
</td>
<td>{{ orphaned.task.start_time }}</td>
<td>
<button type="submit" class="btn btn-sm btn-danger" name="periodic_task_id"
value="{{ orphaned.task.pk }}" form="cancellation-form">Cancel
</button>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<form action="{% url "mooclink.jobs.reset" %}" method="post" id="cancellation-form">
{% csrf_token %}
</form>
{% endblock %}
1 change: 1 addition & 0 deletions pipeline/mooclink/templates/mooclink/course/overview.html
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ <h1 class="mt-2">{{ course.title }}</h1>
{% if perms.subtitles.change_settings %}
<option value="remove-assignment">Remove Assignments</option>
{% endif %}
<option value="download-vtt-files">Download VTT-Files</option>
</select>
<div class="input-group-append">
<button class="btn btn-outline-primary" type="submit">GO</button>
Expand Down
26 changes: 19 additions & 7 deletions pipeline/mooclink/templates/mooclink/course/settings.html
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,16 @@
<div class="container-fluid">
<h1>{{ course.title }}</h1>

<form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}" method="post"
<form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}"
method="post"
id="delete-assigned-language-form">
{% csrf_token %}
<input type="hidden" name="action" value="delete_assigned_language">
</form>

<h2 class="mt-3">Settings</h2>
<form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}" method="post">
<form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}"
method="post">
{% csrf_token %}
<h4>Transcription</h4>
<table>
Expand All @@ -26,7 +28,8 @@ <h4>Transcription</h4>
<td>
<select name="course-language" class="form-control">
{% for lang in available_languages %}
<option value="{{ lang.iso_code }}" {% if lang == course.language %}selected{% endif %}>{{ lang }}</option>
<option value="{{ lang.iso_code }}"
{% if lang == course.language %}selected{% endif %}>{{ lang }}</option>
{% endfor %}
</select>
</td>
Expand All @@ -36,7 +39,13 @@ <h4>Transcription</h4>
{% for transcription_service in supported_translation_services %}
<option value="{{ transcription_service }}"
{% if course.transcription_service == transcription_service %}
selected{% endif %}>{{ transcription_service }}</option>
selected{% endif %}>
{% if transcription_service == "AUDESCR" %}
AUDESCRIBE
{% else %}
{{ transcription_service }}
{% endif %}
</option>
{% endfor %}
</select>
</td>
Expand All @@ -55,7 +64,8 @@ <h4 class="mt-3">Translation</h4>
<tr>
<td>{{ lang.iso_language }}</td>
<td>
<select name="service-provider-{{ lang.pk }}" class="form-control" id="service-provider" required>
<select name="service-provider-{{ lang.pk }}" class="form-control" id="service-provider"
required>
{{ lang.translation_service }}
<option value="">-- Please Select --</option>
{% for translation_service in supported_translation_services %}
Expand All @@ -65,7 +75,8 @@ <h4 class="mt-3">Translation</h4>
{% endfor %}
</select>
</td>
<td><input type="checkbox" class="form-control" name="mandatory-{{ lang.pk }}"{% if lang.required %}
<td><input type="checkbox" class="form-control" name="mandatory-{{ lang.pk }}"
{% if lang.required %}
checked{% endif %}></td>
<td>
<button type="submit" name="delete_asid" value="{{ lang.pk }}" class="btn btn-danger btn-sm"
Expand All @@ -87,7 +98,8 @@ <h4 class="mt-3">Translation</h4>
</table>
<div class="mt-3">
<button type="submit" class="btn btn-primary">Save</button>
<a href="{% tenant_url 'mooclink.course.overview' course_id=course.ext_id tenant=course.tenant %}" class="btn btn-secondary">Back</a>
<a href="{% tenant_url 'mooclink.course.overview' course_id=course.ext_id tenant=course.tenant %}"
class="btn btn-secondary">Back</a>
</div>
</form>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@
Publish
</button>
{% endif %}
<button class="btn btn-outline-primary mr-1" type="submit" name="action" value="download_vtt">Download VTT</button>
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
<div class="row">
<div class="col-2">
<h4>Legend</h4>

<div class="row">
<div class="col-2">
<h4>Legend</h4>
Expand Down
5 changes: 4 additions & 1 deletion pipeline/mooclink/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from mooclink.views.course import CourseOverView, CourseSettingsView, CourseAddLanguageView, RemoveUserAssignment
from mooclink.views.couse_video_bulkaction import CourseBulkActionConfirmation, CourseDoBulkAction, CourseSubscribe
from mooclink.views.main import MainView, MainViewCourse, RedirectByItemId
from mooclink.views.main import MainView, MainViewCourse, RedirectByItemId, JobAdminView, JobAdminResetView
from mooclink.views.service_provider_usage import ServiceProviderUsage, TodoView, ServiceProviderUsageCSV, \
ServiceProviderUsageQuarter
from mooclink.views.subtitle_action import SubtitleToAction
Expand Down Expand Up @@ -37,4 +37,7 @@

# Endpoint for redirecting to course / video by primary-key. Allowed only for admins
path('debug/get_item/', RedirectByItemId.as_view()),

path('jobs/', JobAdminView.as_view(), name="mooclink.jobs.index"),
path('jobs/reset/', JobAdminResetView.as_view(), name="mooclink.jobs.reset"),
]
62 changes: 50 additions & 12 deletions pipeline/mooclink/views/couse_video_bulkaction.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,28 @@
import io
import itertools
import json
import zipfile
from datetime import datetime, timedelta
from operator import itemgetter
from zipfile import ZipFile

from django import views
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin
from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.http import HttpResponse
from django.shortcuts import render, redirect
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils import timezone
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from django.utils.text import slugify
from django_celery_beat.models import IntervalSchedule

from core.exceptions import SecretNotFound
from core.models import Tenant, TranspipeUser
from mooclink.services.aws_translation_service import AwsTranslationService
from mooclink.services.deepl_translation_service import DeeplTranslationService
from mooclink.services.periodic_task_service import PeriodicTaskService
from subtitles.api.xikolo_api import publish_subtitle_to_xikolo
from subtitles.models import Course, Video, IsoLanguage, Subtitle, SubtitleAssignment, AssignedLanguage
from subtitles.models.translation_service import TranslationService
Expand Down Expand Up @@ -274,19 +278,27 @@ def post(self, request, course_id, tenant_slug=None):
period=IntervalSchedule.MINUTES,
)

periodic = PeriodicTask.objects.create(
interval=schedule,
name=f'Check AWS StandaloneTranslation for video={video.pk}',
task='core.tasks.task_update_aws_standalone_translation_status',
# periodic = PeriodicTask.objects.create(
# interval=schedule,
# name=f'Check AWS StandaloneTranslation for video={video.pk}',
# task='core.tasks.task_update_aws_standalone_translation_status',
# start_time=datetime.now() + timedelta(minutes=15),
# kwargs=json.dumps({
# 'tenant_id': video.tenant_id,
# 'video_id': video.pk,
# })
# )

periodic = PeriodicTaskService.create_periodic_task(
service_type="core.tasks.task_update_aws_standalone_translation_status",
task="core.tasks.task_update_aws_standalone_translation_status",
video=video,
start_time=datetime.now() + timedelta(minutes=15),
kwargs=json.dumps({
'tenant_id': video.tenant_id,
'video_id': video.pk,
})
)

video.workflow_data['type'] = 'AWS_TRANSLATION_S_v1'
video.workflow_data['periodic_task_id'] = periodic.pk
if periodic:
video.workflow_data['periodic_task_id'] = periodic.pk
video.workflow_data['initiated'] = str(datetime.utcnow())
video.save()

Expand Down Expand Up @@ -385,6 +397,32 @@ def post(self, request, course_id, tenant_slug=None):
SubtitleAssignment.objects.filter(subtitle=subtitle).update(deleted=timezone.now())

number_of_affected += 1
elif action == "download-vtt-files":
buffer = io.BytesIO()
zip_file = ZipFile(buffer, mode="w", compression=zipfile.ZIP_LZMA)

for video in videos_to_transcript:
with zip_file.open(f"{video.index:02d}_{slugify(video.title)}_{video.original_language.iso_code}.vtt", "w") as f:
if video.current_transcript:
f.write(video.current_transcript.latest_content.encode("utf-8"))

for (language, video) in videos_to_translate:
subtitle = video.subtitle_set.filter(language=language).first()

if not subtitle:
continue

with zip_file.open(f"{video.index:02d}_{slugify(video.title)}_{subtitle.language.iso_code}.vtt", "w") as f:
if video.current_transcript:
f.write(subtitle.latest_content.encode("utf-8"))

zip_file.close()

response = HttpResponse(buffer.getvalue())
response['Content-Type'] = 'application/x-zip-compressed'
response['Content-Disposition'] = f'attachment; filename=subtitles_{slugify(course.title)}.zip'

return response

else:
messages.error(request, f"Action {action} not available")
Expand Down
Loading

0 comments on commit c6033cd

Please sign in to comment.