Add parallel mlp attn option #42

	name: Install Then Test GQA Variations
	on: [push, pull_request]
	jobs:
	Install-And-Test-GQA:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	- run: echo "${{ github.repository }} repository has been cloned to the runner."
	- run: echo "Currently on ${{ github.ref }} branch"
	- name: ls of directory
	run: \|
	ls ${{ github.workspace }}
	# Caching pip dependencies
	- name: Cache pip dependencies
	uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements_cpu.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install CPU Dependencies
	run: \|
	python3 -m pip install --upgrade pip
	python3 -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
	python3 -m pip install numpy transformers datasets tiktoken wandb tqdm tensorboard
	python3 -m pip install -r requirements_cpu.txt
	- name: Test all softmax variations
	run: \|
	python3 data/shakespeare_char/prepare.py
	cd tests
	source test_gqa_variations_cpu.sh

Provide feedback