diff --git a/.env_example b/.env_example index b2f7c33..7381506 100644 --- a/.env_example +++ b/.env_example @@ -1,8 +1,4 @@ SA_API_KEY= S3_ACCESS_KEY= S3_SECRET= -REPO_URL= -GIT_USER= -GIT_EMAIL= -GIT_PASS= -CHANNEL_ID= +CHANNEL_ID= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f6e539e..4e515b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ LABEL description="A Docker container to automatically upload PCC sermons" ENV TZ=America/New_York RUN apk update -RUN apk add python3-dev py3-pip ffmpeg py3-gunicorn git gcc libc-dev libffi-dev tzdata +RUN apk add python3-dev py3-pip ffmpeg py3-gunicorn gcc libc-dev libffi-dev tzdata WORKDIR /app diff --git a/README.md b/README.md index b159bad..5646692 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Generate an API key with python3 main.py -key ``` +or simply edit `data/api.txt`. + Then run the API with ``` @@ -38,9 +40,5 @@ Use the [Docker Compose File](docker-compose.yml) as a guide for filling in all | S3_ACCESS_KEY | Get your access token for your S3 storage if you have it. If not, don't supply it and this part won't run | | S3_SECRET | Same as above, but use your secret token here. You could also use Docker secrets as this is sensitive | | CHANNEL_ID | Supply the _ID_ of the channel you would like to scrape for latest video to scrub through on the web interface. | -| REPO_URL | Put the HTTPS URL for your Git repository here, where a markdown file will be made to upload | -| GIT_USER | Put your Git username here | -| GIT_EMAIL | Put the email you want the committer to use on Git here | -| GIT_PASS | Personal Access Token or password for your Git provider. If using Github you can obtain one [here](https://github.com/settings/tokens/new) | Enjoy! If anyone else ever tries to use this it'll need to be customized a great deal, but the upload pipelines should be ironed out at the very least. Shoot me a message if you have any problems, although the only person who will probably have any issues will be me ;) diff --git a/classes.py b/classes.py index db746a7..617a6ee 100644 --- a/classes.py +++ b/classes.py @@ -3,11 +3,14 @@ import subprocess import time from datetime import datetime +from datetime import timedelta + import requests from dateutil.parser import parse from dotenv import load_dotenv from pdfminer.high_level import extract_text +from youtube_transcript_api import YouTubeTranscriptApi from pydub import AudioSegment from yt_dlp import YoutubeDL @@ -40,7 +43,7 @@ class Sermon: # video: current location of video file # filename: final pretty filename, ends in mp. Tack on 3 or 4 - # Initialize sermon and fetch pertinent info + # Constructor for initializing a Sermon object def __init__(self, payload): self.date = payload["date"] try: @@ -55,6 +58,7 @@ def __init__(self, payload): self.youtube = payload["youtube"] self.website = payload["website"] except: + # Set attributes to None or False if not provided in the payload self.title = None self.speaker = None self.text = None @@ -68,11 +72,14 @@ def __init__(self, payload): pass self.audio = None self.video = None + # Create sermon if no title or videoId is provided if not self.title or not self.videoId: self.make() + # Method to generate missing sermon information def make(self): pdf_url = None + # Request bulletin data response = requests.get( "https://coventrypca.church/assets/data/bulletins/index.json" ) @@ -132,7 +139,8 @@ def make(self): self.text = passage self.speaker = speaker - # Informative functions + # MARK: Info methods + # Method to check if the sermon has already been uploaded def isUploaded(self): try: sermons = [] @@ -168,7 +176,77 @@ def isUploaded(self): self.uploaded = False return False - # Actions: + # Method to estimate the start and end time of the sermon from the transcript + def guessTiming(self): + if not self.videoId: + self.start = "0:20:00" + self.end = "0:50:00" + return + try: + transcript = YouTubeTranscriptApi.get_transcript(self.videoId) + except Exception: + self.start = "0:20:00" + self.end = "0:50:00" + return + if not transcript: + self.start = "0:20:00" + self.end = "0:50:00" + return + events = [] + for i in range(len(transcript) - 1): + current_start = timedelta(seconds=transcript[i]["start"]) + combined_text = transcript[i]["text"] + " " + transcript[i + 1]["text"] + events.append((current_start, combined_text)) + for i in range(len(events) - 1): + time, text = events[i] + if "amen please be" in text: + start_time = time + # elif "please be seated" in text: + # start_time = time + elif "name amen" in text and start_time: + if time - start_time >= timedelta( + minutes=20 + ) and time - start_time <= timedelta(minutes=40): + intermediate_texts = [ + txt for t, txt in events if start_time < t < time + ] + intermediate_texts.pop(0) + if not any("please be seated" in txt for txt in intermediate_texts): + end_time = events[i + 1][0] + break + # elif "let's stand" in text and start_time: + # if time - start_time >= timedelta(minutes=20) and time - start_time <= timedelta(minutes=40): + # intermediate_texts = [txt for t, txt in events if start_time < t < time] + # intermediate_texts.pop(0) + # if not any("please be seated" in txt for txt in intermediate_texts): + # end_time = events[i - 1][0] + # break + elif "closing hymn" in text and start_time: + if time - start_time >= timedelta( + minutes=20 + ) and time - start_time <= timedelta(minutes=45): + intermediate_texts = [ + txt for t, txt in events if start_time < t < time + ] + intermediate_texts.pop(0) + if not any("please be seated" in txt for txt in intermediate_texts): + end_time = events[i - 2][0] + break + if start_time and end_time: + self.start = start_time + self.end = end_time + return + elif start_time: + self.start = start_time + self.end = "0:50:00" + return + else: + self.start = "0:20:00" + self.end = "0:50:00" + return + + # MARK: Action methods + # Method to download the sermon video def download(self): self.filename = ( "process/" @@ -207,6 +285,7 @@ def download(self): else: self.video = self.rawVideo + # Method to trim the downloaded video to the sermon portion def trimVideo(self): log("Trimming livestream...") try: @@ -237,6 +316,7 @@ def trimVideo(self): except: log("❌\n") + # Method to process the sermon audio with noise reduction def processAudio(self): # Run noisereduction on provided file: if not self.audio: @@ -276,6 +356,7 @@ def processAudio(self): except: log("❌\n") + # Method to convert video to audio def videoToAudio(self): log("Converting video to audio...") try: @@ -289,6 +370,7 @@ def videoToAudio(self): except: log("❌\n") + # Method to upload the sermon to various platforms def upload(self): self.download() if self.sermonAudio or self.website: diff --git a/examples/Upload.vue b/examples/Upload.vue index 333023f..0c03fb3 100644 --- a/examples/Upload.vue +++ b/examples/Upload.vue @@ -124,7 +124,7 @@ hide-details single-line type="number" - style="width: 60px;" + style="width: 60px" @change="$set(form.value, 0, $event), scrubVideo()" > @@ -135,7 +135,7 @@ hide-details single-line type="number" - style="width: 60px;" + style="width: 60px" @change="$set(form.value, 1, $event), scrubVideo()" > @@ -189,9 +189,7 @@ color="error" @click="reFetch()" > - - mdi-alert - + mdi-alert Server Unavailable - Reload @@ -200,11 +198,11 @@ diff --git a/main.py b/main.py index 6063058..d782e7b 100644 --- a/main.py +++ b/main.py @@ -19,7 +19,6 @@ from time import sleep from types import SimpleNamespace -import git import requests from colorama import Fore, Style from dateutil.parser import parse @@ -101,19 +100,14 @@ def setup(mode): # Get list of speakers and series to pick from in the Vue frontend def getSpeakers(): - # Needed stuff - repo_url = os.environ["REPO_URL"] - repo_path = repo_url.split("/")[-1] - # Remove existing repo if it still exists instead of risking a pull - os.system("rm -rf " + repo_path) - # Clone repo from remote - git.Repo.clone_from(repo_url, repo_path) - speakers = os.listdir(repo_path + "/content/preachers") + url = f"https://api.github.com/repos/Presbyterian-Church-of-Coventry/pcc-website/contents/content/preachers" + response = requests.get(url) preachers = [] - for speaker in speakers: - preachers.append(speaker[:-3].replace("-", " ").title()) - # Clean up for next time - os.system("rm -rf " + repo_path) + if response.status_code == 200: + content = response.json() + for file in content: + if file["type"] == "file": + preachers.append(file["name"][:-3].replace("-", " ").title()) with open("data/speakers.txt", "w") as f: f.write(str(preachers)) f.close() @@ -283,6 +277,8 @@ def get_sermon(sermon_date): "date": sermon.date, "videoId": sermon.videoId, "uploaded": sermon.uploaded, + "guessStart": sermon.start, + "guessEnd": sermon.end, }, 200 @@ -347,7 +343,7 @@ def posted(): def serveAPI(): print(Fore.GREEN + "Started serving API on port 3167!" + Fore.RESET) os.system( - "gunicorn -b 0.0.0.0:3167 main:app --workers=3 --enable-stdio-inheritance --error-logfile logs/error.log --access-logfile logs/access.log --log-level info -e GIT_PYTHON_GIT_EXECUTABLE=/usr/bin/git" + "gunicorn -b 0.0.0.0:3167 main:app --workers=3 --enable-stdio-inheritance --error-logfile logs/error.log --access-logfile logs/access.log --log-level info" ) diff --git a/requirements.txt b/requirements.txt index 0be5f6f..0f928d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ typing==3.7.4.3 six==1.16.0 datetime==5.0 python-dateutil==2.8.2 +youtube_transcript_api==0.6.2 diff --git a/upload.py b/upload.py index 14798bf..1e74967 100644 --- a/upload.py +++ b/upload.py @@ -168,7 +168,9 @@ def getAuthenticatedService(): credentials = pickle.load(token) else: # If not, perform OAuth 2.0 authorization flow - flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(client_secrets_file, scopes) + flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file( + client_secrets_file, scopes + ) credentials = flow.run_console() with open("data/token.pickle", "wb") as token: pickle.dump(credentials, token)