Merge pull request #10 from MikroElektronika/release-workflow

Release workflow

.github/workflows/index.yaml

@@ -0,0 +1,33 @@
+name: Index Latest Release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  index:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install aiohttp
+          pip install aiofiles
+          pip install requests
+          pip install elasticsearch==7.13.4
+          sudo apt-get install p7zip-full
+
+      - name: Run Index Script
+        env:
+          ES_HOST: ${{ secrets.ES_HOST }}
+          ES_INDEX: ${{ secrets.ES_INDEX }}
+          ES_USER: ${{ secrets.ES_USER }}
+          ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
+        run: python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -0,0 +1,26 @@
+name: Release Asset Upload
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  upload-release-asset:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+          pip install py7zr
+
+      - name: Run Release Script
+        run: python -u scripts/package.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }}

.gitignore

@@ -0,0 +1,6 @@
+*.asm
+.vscode
+output
+__pycache__
+*.7z
+tmp

scripts/index.py

@@ -0,0 +1,132 @@
+import os, time, argparse, requests
+from elasticsearch import Elasticsearch
+
+import support as support
+
+# Gets latest release headers from repository
+def get_headers(api, token):
+    if api:
+        return {
+            'Authorization': f'token {token}'
+        }
+    else:
+        return {
+            'Authorization': f'Bearer {token}',
+            'Accept': 'application/octet-stream'
+        }
+
+# Function to fetch release details from GitHub
+def fetch_release_details(repo, token):
+    api_headers = get_headers(True, token)
+    url = f'https://api.github.com/repos/{repo}/releases'
+    response = requests.get(url, headers=api_headers)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+    return support.get_latest_release(response.json())
+
+# Function to fetch content as JSON from the link
+def fetch_json_data(download_link, token):
+    """
+    Fetches JSON data from the specified URL using an authorization token and returns it as a dictionary.
+
+    Args:
+        download_link (str): URL from which to fetch the JSON data.
+
+    Returns:
+        tuple: The first element is a dictionary containing the JSON data (or None in case of failure),
+                the second element is an error message or None if no errors occurred.
+    """
+    headers = get_headers(False, token)
+
+    try:
+        response = requests.get(download_link, headers=headers)
+        response.raise_for_status()
+        return response.json(), None
+    except requests.RequestException as e:
+        print(f"Error fetching JSON data: {e}")
+        return None, str(e)
+
+# Function to find an item by name
+def find_item_by_name(items, name):
+    for item in items:
+        if item['name'] == name:
+            return item
+    return None
+
+# Function to index release details into Elasticsearch
+def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_details, token):
+    version = None
+    for asset in release_details.get('assets', []):
+        if 'mikrosdk.7z' == asset['name']:
+            # Download the current mikroSDK asset in order to read the version
+            support.extract_archive_from_url(
+                asset['url'],
+                os.path.join(os.path.dirname(__file__), 'tmp'),
+                token
+            )
+
+            # Then fetch version from manifest file
+            version = support.fetch_version_from_asset(os.path.join(os.path.dirname(__file__), 'tmp'))
+            break
+    for asset in release_details.get('assets', []):
+        doc = None
+        name_without_extension = os.path.splitext(os.path.basename(asset['name']))[0]
+        if 'mikrosdk' == name_without_extension:
+            doc = {
+                'name': name_without_extension,
+                'display_name': "mikroSDK",
+                'author': 'MIKROE',
+                'hidden': False,
+                'type': 'sdk',
+                'version': version,
+                'created_at' : asset['created_at'],
+                'updated_at' : asset['updated_at'],
+                'category': 'Software Development Kit',
+                'download_link': asset['url'],  # Adjust as needed for actual URL
+                'package_changed': True
+            }
+        elif 'templates' == name_without_extension:
+            doc = {
+                "name": name_without_extension,
+                "version" : version,
+                "display_name" : "NECTO project templates",
+                "hidden" : True,
+                "vendor" : "MIKROE",
+                "type" : "application",
+                "download_link" : asset['url'],
+                "install_location" : "%APPLICATION_DATA_DIR%/templates",
+                "package_changed": True
+            }
+
+        # Index the document
+        if doc:
+            resp = es.index(index=index_name, doc_type='necto_package', id=name_without_extension, body=doc)
+            print(f"{resp["result"]} {resp['_id']}")
+
+if __name__ == '__main__':
+    # Get arguments
+    parser = argparse.ArgumentParser(description="Upload directories as release assets.")
+    parser.add_argument("repo", help="Repository name, e.g., 'username/repo'")
+    parser.add_argument("token", help="GitHub Token")
+    args = parser.parse_args()
+
+    # Elasticsearch instance used for indexing
+    num_of_retries = 1
+    while True:
+        print(f"Trying to connect to ES. Connection retry:  {num_of_retries}")
+        es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD']))
+        if es.ping():
+            break
+        # Wait for 30 seconds and try again if connection fails
+        if 10 == num_of_retries:
+            # Exit if it fails 10 times, something is wrong with the server
+            raise ValueError("Connection to ES failed!")
+        num_of_retries += 1
+
+        time.sleep(30)
+
+    # Now index the new release
+    index_release_to_elasticsearch(
+        es, os.environ['ES_INDEX'],
+        fetch_release_details(args.repo, args.token),
+        args.token
+    )

scripts/package.py

@@ -0,0 +1,84 @@
+import os, re, py7zr, requests, argparse, json
+
+def find_manifest_folder(base_dir):
+    """Find the folder containing 'manifest.json'."""
+    for root, dirs, files in os.walk(base_dir):
+        if 'manifest.json' in files:
+            return root
+    return None
+
+def create_7z_archive(version, source_folder, archive_path):
+    """Create a .7z archive from a source folder with a specific folder structure, excluding the .github folder."""
+    with py7zr.SevenZipFile(archive_path, 'w') as archive:
+        for root, dirs, files in os.walk(source_folder):
+            if re.search(r'(\.git)|(scripts)|(templates)', os.path.relpath(root, source_folder)):
+                continue
+            for file in files:
+                file_path = os.path.join(root, file)
+                # Exclude the archive itself
+                if file_path == archive_path:
+                    continue
+                archive.write(file_path, os.path.join(version, 'src', os.path.relpath(file_path, source_folder)))
+
+def create_template_archive(source_folder, archive_path):
+    """Create a .7z archive from a source folder with a specific folder structure."""
+    with py7zr.SevenZipFile(archive_path, 'w') as archive:
+        os.chdir(source_folder)
+        archive.writeall('./')
+        os.chdir('..')
+
+def upload_asset_to_release(repo, release_id, asset_path, token):
+    """Upload an asset to a specific GitHub release."""
+    url = f'https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={os.path.basename(asset_path)}'
+    headers = {
+        'Authorization': f'token {token}',
+        'Content-Type': 'application/x-7z-compressed'
+    }
+    with open(asset_path, 'rb') as file:
+        response = requests.post(url, headers=headers, data=file)
+        response.raise_for_status()
+        print(f'Uploaded asset: {os.path.basename(asset_path)} to release ID: {release_id}')
+        return response.json()
+
+def get_release_id(repo, tag_name, token):
+    """Get the release ID for a given tag name."""
+    url = f'https://api.github.com/repos/{repo}/releases/tags/{tag_name}'
+    headers = {'Authorization': f'token {token}'}
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.json()['id']
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Upload directories as release assets.")
+    parser.add_argument("token", help="GitHub Token")
+    parser.add_argument("repo", help="Repository name, e.g., 'username/repo'")
+    args = parser.parse_args()
+
+    # Assuming the repository is checked out at the root directory
+    repo_dir = os.getcwd()
+    manifest_folder = find_manifest_folder(repo_dir)
+    version = json.load(open(os.path.join(manifest_folder ,'manifest.json')))['sdk-version']
+
+    if manifest_folder:
+        archive_path = os.path.join(repo_dir, 'mikrosdk.7z')
+        print('Creating archive: %s' % archive_path)
+        create_7z_archive(version, repo_dir, archive_path)
+        print('Archive created successfully: %s' % archive_path)
+
+        # Get the release ID and upload the asset
+        release_id = get_release_id(args.repo, f'mikroSDK-{version}', args.token)
+        upload_result = upload_asset_to_release(args.repo, release_id, archive_path, args.token)
+
+        print('Asset "%s" uploaded successfully to release ID: %s' % ('mikrosdk', release_id))
+
+    if os.path.exists(os.path.join(repo_dir, 'templates')):
+        archive_path = os.path.join(repo_dir, 'templates.7z')
+        print('Creating archive: %s' % archive_path)
+        create_template_archive('templates', archive_path)
+        print('Archive created successfully: %s' % archive_path)
+
+        # Get the release ID and upload the asset
+        release_id = get_release_id(args.repo, f'mikroSDK-{version}', args.token)
+        upload_result = upload_asset_to_release(args.repo, release_id, archive_path, args.token)
+
+        print('Asset "%s" uploaded successfully to release ID: %s' % ('templates', release_id))

scripts/support.py

@@ -0,0 +1,79 @@
+import requests, py7zr, zipfile, io, os, json
+
+def get_previous_release(releases, prerelases=None):
+    ''' Fetch the previously released version '''
+    for counter, release in enumerate(releases):
+        if not release['draft']:
+            if prerelases:
+                if release['prerelease']:
+                    continue
+            if counter + 1 < len(releases):
+                return releases[counter + 1]
+            else:
+                return None
+    return None
+
+def get_latest_release(releases):
+    ''' Fetch the latest released version '''
+    return next((release for release in releases if not release['prerelease'] and not release['draft']), None)
+
+def determine_archive_type(byte_stream):
+    '''
+    Implement logic to determine the archive type, e.g., by file extension or magic number
+    For simplicity, let's assume byte_stream has a 'name' attribute (e.g., a file-like object)
+    '''
+    byte_stream.seek(0)
+    signature = byte_stream.read(4)
+    byte_stream.seek(0)
+    if signature == b'PK\x03\x04':  # ZIP magic number, it is what it is
+        return 'zip'
+    else:
+        return '7z'
+
+def extract_archive_from_url(url, destination, token):
+    """
+    Extract the contents of an archive (7z or zip) from a URL directly
+    in memory, without downloading the file.
+    """
+    print(f"Download link: {url}")
+    headers = {
+        'Authorization': f'token {token}',
+        'Accept': 'application/octet-stream'
+    }
+    if 'github' in url:
+        response = requests.get(url, headers=headers, stream=True)
+    else:
+        response = requests.get(url, stream=True)
+
+    response.raise_for_status()
+
+    if response.status_code == 200: ## Response OK?
+        with io.BytesIO() as byte_stream:
+
+            for chunk in response.iter_content(chunk_size=8192):
+                byte_stream.write(chunk)
+
+            byte_stream.seek(0)
+
+            archive_type = determine_archive_type(byte_stream)
+
+            if archive_type == '7z':
+                with py7zr.SevenZipFile(byte_stream, mode='r') as archive:
+                    archive.extractall(path=destination)
+            elif archive_type == 'zip':
+                with zipfile.ZipFile(byte_stream, mode='r') as archive:
+                    for info in archive.infolist():
+                        archive.extract(info, path=destination)
+            else:
+                raise ValueError("Unsupported archive type")
+    else:
+        raise Exception(f"Failed to download file: status code {response.status_code}")
+
+def fetch_version_from_asset(asset_path):
+    for root, subdirs, files in os.walk(asset_path):
+        for file_name in files:
+            if 'manifest.json' == file_name:
+                return json.load(open(os.path.join(root, file_name)))['sdk-version']
+            else:
+                continue
+    return None

templates/designer_generated_code/codegen_screen.c

@@ -0,0 +1,21 @@
+/*This is a comment.*/
+
+#include "vtft.h"
+#include <string.h>
+#include <assembly.h>
+#include "resource.h"
+#include "scr_%1.h"
+
+%2
+
+void initialize_%1() 
+{
+    %3
+}
+
+int get_%1_initialized() 
+{
+    return %1_initialized; 
+}
+
+