diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..5e30128b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +Flatpak_Test_App/flatpak/build-dir +Flatpak_Test_App/flatpak/.flatpak-builder diff --git a/Flatpak_Test_App/Readme.md b/Flatpak_Test_App/Readme.md new file mode 100644 index 000000000..8796173de --- /dev/null +++ b/Flatpak_Test_App/Readme.md @@ -0,0 +1,50 @@ +# Flatpak_Test_App (FOSSEE_Installer_Test) WIP :warning: + + +## Documentation to package demo installer using Flatpak. + +### To get stared, we need to first install the runtimes required for packing our Qt5 application + +``` + flatpak install org.kde.Sdk//5.15-22.08 + + flatpak install com.riverbankcomputing.PyQt.BaseApp//5.15-22.08 +``` +--- + +### To build the flatpak application we require flatpak-builder + +Ubuntu: +``` + sudo apt install flatpak-builder -y +``` +Fedora: +``` + sudo dnf install flatpak-builder -y +``` + +### Install the dependencies mentioned below on the host system + +Ubuntu: +``` + sudo apt install make flex g++ ccache bison -y +``` +Fedora: +``` + sudo dnf install make flex g++ ccache bison -y +``` +Other Linux Distributions: Use equivalent commands to install the necesarry packages + +### To see all available commands for building, installing, accessing runtime through shell and more... +```bash + # execute the listed commands in ./flatpak/ dir + bash scripts/build_new_flatpak.sh +``` + +## Screenshots: +![image](https://user-images.githubusercontent.com/75079303/236667264-3d77cd7a-9bf0-405b-a0fc-32cde1f2f9be.png) + +![image](https://user-images.githubusercontent.com/75079303/236667291-1467a423-d1d1-479d-b2f2-3fd49e195926.png) + +![image](https://user-images.githubusercontent.com/75079303/236675064-114cffc4-f05f-4f7d-9e33-252cc7e062ef.png) + diff --git a/Flatpak_Test_App/flatpak/FOSSEE_Inst_test.py b/Flatpak_Test_App/flatpak/FOSSEE_Inst_test.py new file mode 100644 index 000000000..2ba05c266 --- /dev/null +++ b/Flatpak_Test_App/flatpak/FOSSEE_Inst_test.py @@ -0,0 +1,147 @@ +import sys +import subprocess +from PyQt5 import QtWidgets +from PyQt5.QtWidgets import QMessageBox +import PyQt5.QtCore +from PyQt5.QtCore import * +from PyQt5.QtGui import QPixmap +from PyQt5.QtWidgets import QLabel +import py7zr +import os + +class AppWindow(QtWidgets.QWidget): + def __init__(self): + super().__init__() + self.initUI() + + def initUI(self): + self.Logo = QLabel(self) + self.pixmap = QPixmap('/app/assets/Fossee_logo.png') + self.Logo.setPixmap(self.pixmap) + self.Logo.setAlignment(PyQt5.QtCore.Qt.AlignRight) + + self.label = QtWidgets.QLabel('Select package to install:') + self.combo = QtWidgets.QComboBox() + self.combo.addItems(['makerchip-app', 'sandpiper-saas', 'matplotlib']) + + self.install_pkg_button = QtWidgets.QPushButton('Install Pip package') + self.install_pkg_button.clicked.connect(self.install_Pip_package) + + self.install_verilator_from_Archive_button = QtWidgets.QPushButton('Install Verilator from Archive') + self.install_verilator_from_Archive_button.clicked.connect(self.intall_Verilator_from_Archive) + + self.install_verilator_SRC_button = QtWidgets.QPushButton('Install Verilator from SRC') + self.install_verilator_SRC_button.clicked.connect(self.install_Verilator_SRC) + + self.Install_Output = QtWidgets.QPlainTextEdit() + + self.install_thread = InstallThread_SRC() + self.install_thread.output_signal.connect(self.update_output) + + layout = QtWidgets.QVBoxLayout() + layout.addWidget(self.Logo) + layout.addWidget(self.label) + layout.addWidget(self.combo) + layout.addWidget(self.install_pkg_button) + layout.addWidget(self.install_verilator_from_Archive_button) + layout.addWidget(self.install_verilator_SRC_button) + layout.addWidget(self.Install_Output) + self.setLayout(layout) + + def install_Pip_package(self): + package = self.combo.currentText() + try: + subprocess.check_call(['flatpak-spawn', '--host', sys.executable, '-m', 'pip', 'install', package]) + QtWidgets.QMessageBox.information(self, 'Success', f'{package} installed successfully') + self.Install_Output.appendPlainText(f'Success: {package} installed successfully') + except Exception as e: + QMessageBox.warning(self,"ERROR", str(e)) + self.Install_Output.appendPlainText("[ERROR!]: ") + self.Install_Output.appendPlainText(str(e)) + + def intall_Verilator_from_Archive(self): + file_name = "/app/assets/packages/verilator.7z" + output_dir = os.getcwd() + + source_file = f"{output_dir}/verilator/bin/verilator" + destination_dir = "/usr/bin/" + + with py7zr.SevenZipFile(file_name, mode='r') as z: + z.extractall(path=output_dir) + + try: + subprocess.check_call(['flatpak-spawn', '--host', "pkexec", "cp", source_file, destination_dir]) + QtWidgets.QMessageBox.information(self, 'Success', 'Verilator installed successfully from Archive') + self.Install_Output.appendPlainText('[Success]: verilator installed successfully from Archive') + except subprocess.CalledProcessError as e: + QMessageBox.warning(self,"ERROR", str(e)) + self.Install_Output.appendPlainText("[ERROR!]: ") + self.Install_Output.appendPlainText(str(e)) + + ## Used to show updates for install_Verilator_SRC + def update_output(self, text): + self.Install_Output.appendPlainText(text) + + def install_Verilator_SRC(self): + try: + self.Install_Output.clear() + + self.Install_Output.clear() + self.install_thread.start() + + except Exception as e: + self.Install_Output.appendPlainText("[ERROR!]: ") + self.Install_Output.appendPlainText(str(e)) + QtWidgets.QMessageBox.information(self,"Error", str(e)) + print("[ERROR!]: ",e) + +class install_verilator(QThread): + P_progress = pyqtSignal(str) + def __init__(self, command, parent=None): + QThread.__init__(self, parent) + self.command = command + def start(self): + QThread.start(self) + def run(self): + try: + p = subprocess.run(str(self.command),shell=True,capture_output=True) + status = p.stdout.decode() + if p: + print(status) + self.P_progress.emit(status) + except Exception as e: + self.Install_Output.appendPlainText("[ERROR!]: ") + self.Install_Output.appendPlainText(str(e)) + QMessageBox.warning(None,"Invalid Install Commands!","Error in install_verilator") + +class InstallThread_SRC(QThread): + output_signal = pyqtSignal(str) + + def __init__(self): + super().__init__() + + def run(self): + command = "/app/assets/scripts/Install_verilator.sh" + process = subprocess.Popen(["bash", command], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + while True: + output = process.stdout.readline() + if output == b'' and process.poll() is not None: + break + if output: + self.output_signal.emit(str(output.strip())[2:-1]) + print(output.strip()[2:-1]) + + rc = process.poll() + if rc == 0: + self.output_signal.emit("[Info] Installation succeeded!") + print("Installation succeeded!") + else: + self.output_signal.emit("[ERROR] Installation failed!") + print("Installation failed!") + +if __name__ == '__main__': + app = QtWidgets.QApplication(sys.argv) + ex = AppWindow() + ex.show() + sys.exit(app.exec_()) \ No newline at end of file diff --git a/Flatpak_Test_App/flatpak/assets/Fossee_logo.png b/Flatpak_Test_App/flatpak/assets/Fossee_logo.png new file mode 100644 index 000000000..ecd7e44a6 Binary files /dev/null and b/Flatpak_Test_App/flatpak/assets/Fossee_logo.png differ diff --git a/Flatpak_Test_App/flatpak/assets/packages/verilator.7z b/Flatpak_Test_App/flatpak/assets/packages/verilator.7z new file mode 100644 index 000000000..bd9f5c524 Binary files /dev/null and b/Flatpak_Test_App/flatpak/assets/packages/verilator.7z differ diff --git a/Flatpak_Test_App/flatpak/assets/scripts/Install_verilator.sh b/Flatpak_Test_App/flatpak/assets/scripts/Install_verilator.sh new file mode 100755 index 000000000..7dddc1746 --- /dev/null +++ b/Flatpak_Test_App/flatpak/assets/scripts/Install_verilator.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +## Clone Verilator repository + +echo '# Please wait... cloning https://github.com/verilator/verilator' +git clone http://git.veripool.org/git/verilator +echo '# repo cloned...' +echo "===============================================\n" + +echo '# changing dir to verilator/' +cd ./verilator + +echo '# current working directory is: ' +pwd +echo "===============================================\n" + +# Switch to version 4.106 +git checkout v4.106 + +# Build Verilator +autoconf + +echo '# please wait... This may take some time' +echo "===============================================\n" +./configure --prefix=/usr --libdir=/usr/lib64 + +echo '# current working directory is (supposed to be verilator): ' +pwd +echo "===========================" +echo "Please enter root password:" +flatpak-spawn --host sudo -S make + + +echo 'Installing make...' +echo "===========================" +echo "Please enter root password:" +flatpak-spawn --host sudo -S make install + +flatpak-spawn --host verilator --version + +echo "===============================================\n" +echo "Cleaning up..." + +flatpak-spawn --host rm -rf ./verilator \ No newline at end of file diff --git a/Flatpak_Test_App/flatpak/assets/scripts/ngspice.sh b/Flatpak_Test_App/flatpak/assets/scripts/ngspice.sh new file mode 100644 index 000000000..5710faef6 --- /dev/null +++ b/Flatpak_Test_App/flatpak/assets/scripts/ngspice.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +echo "Installing ngspice..." +echo "===========================" + +# write a script to install ngspice on any platform + +# Download ngspice source code from GitHub +git clone https://github.com/ngspice/ngspice.git + +# Change directory to ngspice folder +cd ngspice + +# Install prerequisites + +# Configure and build ngspice +echo '# please wait... This may take some time' +echo "===============================================\n" +./autogen.sh +./configure +make +sudo make install + +# Add ngspice executable to PATH environment variable +echo 'export PATH=$PATH:/usr/local/bin' >> ~/.bashrc +source ~/.bashrc + +# Test ngspice installation +ngspice --version \ No newline at end of file diff --git a/Flatpak_Test_App/flatpak/assets/scripts/test b/Flatpak_Test_App/flatpak/assets/scripts/test new file mode 100755 index 000000000..fbf3b16b2 --- /dev/null +++ b/Flatpak_Test_App/flatpak/assets/scripts/test @@ -0,0 +1,10 @@ +#!/bin/sh + +echo "This is a test script" +echo "========================================" +git --version +echo "========================================" +make --version +echo "========================================" +autoconf --verison +echo "========================================" diff --git a/Flatpak_Test_App/flatpak/git.json b/Flatpak_Test_App/flatpak/git.json new file mode 100644 index 000000000..20770f63d --- /dev/null +++ b/Flatpak_Test_App/flatpak/git.json @@ -0,0 +1,26 @@ +{ + "name": "git", + "make-args": [ + "INSTALL_SYMLINKS=1" + ], + "make-install-args": [ + "INSTALL_SYMLINKS=1" + ], + "cleanup": [ + "/lib/pkgconfig", + "/man" + ], + "sources": [ + { + "type": "archive", + "url": "https://www.kernel.org/pub/software/scm/git/git-2.40.1.tar.xz", + "sha256": "4893b8b98eefc9fdc4b0e7ca249e340004faa7804a433d17429e311e1fef21d2", + "x-checker-data": { + "type": "anitya", + "project-id": 5350, + "stable-only": true, + "url-template": "https://www.kernel.org/pub/software/scm/git/git-$version.tar.xz" + } + } + ] +} \ No newline at end of file diff --git a/Flatpak_Test_App/flatpak/org.flatpak.FOSSEE_Inst_test.yml b/Flatpak_Test_App/flatpak/org.flatpak.FOSSEE_Inst_test.yml new file mode 100644 index 000000000..fac2bffde --- /dev/null +++ b/Flatpak_Test_App/flatpak/org.flatpak.FOSSEE_Inst_test.yml @@ -0,0 +1,49 @@ +app-id: org.flatpak.FOSSEE_Inst_test +runtime: org.kde.Sdk +runtime-version: '5.15-22.08' +sdk: org.kde.Sdk +base: com.riverbankcomputing.PyQt.BaseApp +base-version: 5.15-22.08 + +command: run.sh + +finish-args: + - --share=ipc + - --share=network + - --socket=x11 + - --socket=wayland + - --filesystem=host-os + - --filesystem=host + - --filesystem=home + - --device=dri + - --talk-name=org.freedesktop.Flatpak + +files: + - path: /home/suchinton/verilator/* + read: true + write: true + +modules: + - name: FOSSEE_Inst_test + buildsystem: simple + build-commands: + - install -D FOSSEE_Inst_test.py /app/bin/FOSSEE_Inst_test.py + - install -D run.sh /app/bin/run.sh + + sources: + - type: file + path: FOSSEE_Inst_test.py + - type: file + path: run.sh + + - name: assets + buildsystem: simple + build-commands: + - cp -r assets/ /app/assets/ + + sources: + - type: dir + path: assets/ + dest: assets/ + + - python3-py7zr.json \ No newline at end of file diff --git a/Flatpak_Test_App/flatpak/python3-py7zr.json b/Flatpak_Test_App/flatpak/python3-py7zr.json new file mode 100644 index 000000000..392c02efe --- /dev/null +++ b/Flatpak_Test_App/flatpak/python3-py7zr.json @@ -0,0 +1,59 @@ +{ + "name": "python3-py7zr", + "buildsystem": "simple", + "build-commands": [ + "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"py7zr\" --no-build-isolation" + ], + "sources": [ + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/2a/18/70c32fe9357f3eea18598b23aa9ed29b1711c3001835f7cf99a9818985d0/Brotli-1.0.9.zip", + "sha256": "4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/14/6f/825068e229aaa59719d252d2279a23ba471f95d6df18ca09e9f7f445fda3/inflate64-0.3.1.tar.gz", + "sha256": "b52dd8fefd2ba179e5dfa18d6eca7e2fc822584616271c039d5ef1f9ca90c71c" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/22/31/ec5f46fd4c83185b806aa9c736e228cb780f13990a9cf4da0beb70025fcc/multivolumefile-0.2.3-py3-none-any.whl", + "sha256": "237f4353b60af1703087cf7725755a1f6fcaeeea48421e1896940cd1c920d678" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/3d/7d/d05864a69e452f003c0d77e728e155a89a2a26b09e64860ddd70ad64fb26/psutil-5.9.4.tar.gz", + "sha256": "3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/1b/9c/514791abe077098ca6b4dac2ba8780c2f162aa35b5d99e48915a0f4ad2a8/py7zr-0.20.4-py3-none-any.whl", + "sha256": "94d0c24217f6582741813ee94490a4ca82bd5f9bf35e4f8610cb588cf7445764" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/26/ff/25952179fa892e7d082a34e5917d747d05a227ba3c83dd3adac1c3f0be24/pybcj-1.0.1.tar.gz", + "sha256": "8b682ed08caabfb7c042d4be083e28ddc692afb1deff5567111f8855071b75c3" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/3d/07/cfd8f52b9068877801317d26dc7225e19421bc659e1395d2cd6933b1a351/pycryptodomex-3.17.tar.gz", + "sha256": "0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/49/93/e45d3f8d2725ec448b9178b91791c2509d3a9b42d8984a22f576fe2f89be/pyppmd-1.0.0.tar.gz", + "sha256": "075c9bd297e3b0a87dd7aeabca7fee668218acbe69ecc1c6511064558de8840f" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/b0/d8/75d91aa30bac7fa5b88f7216768c932b8b7ad48ccff1d2e322655c0571cd/pyzstd-0.15.4.tar.gz", + "sha256": "de07ac54f57642f186732075cdce2be3d4a30228c3b17a6d8c6053765dc6eec8" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/ba/a7/2c12b543f853dae886286b824200eb9d7cd2466e3d14eff1799fbe8223b9/texttable-1.6.7-py2.py3-none-any.whl", + "sha256": "b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c" + } + ] +} diff --git a/Flatpak_Test_App/flatpak/run.sh b/Flatpak_Test_App/flatpak/run.sh new file mode 100644 index 000000000..5e0621db6 --- /dev/null +++ b/Flatpak_Test_App/flatpak/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +python3 -u /app/bin/FOSSEE_Inst_test.py diff --git a/Flatpak_Test_App/req b/Flatpak_Test_App/req new file mode 100644 index 000000000..9bcc1755e --- /dev/null +++ b/Flatpak_Test_App/req @@ -0,0 +1,2 @@ +py7zr +pyqt5 \ No newline at end of file diff --git a/Flatpak_Test_App/scripts/build_new_flatpak.sh b/Flatpak_Test_App/scripts/build_new_flatpak.sh new file mode 100755 index 000000000..d41639847 --- /dev/null +++ b/Flatpak_Test_App/scripts/build_new_flatpak.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +echo "Reference Commands:" +echo "=========================================================================" +echo "To build, run command:" +echo " $ flatpak-builder --force-clean build-dir org.flatpak.FOSSEE_Inst_test.yml" +echo "=========================================================================" +echo "To install the app, run command:" +echo " $ flatpak-builder --user --install --force-clean build-dir org.flatpak.FOSSEE_Inst_test.ym" +echo "=========================================================================" +echo "To run application, run command:" +echo ' $ flatpak run org.flatpak.FOSSEE_Inst_test to run app' +echo "=========================================================================" +echo "To resolove new pip dependencies, run command:" +echo ' $ python3 flatpak-pip-generator --requirements-file='$HOME/Repos/Test/req' --output pypi-dependencies --yaml' +echo "=========================================================================" diff --git a/Flatpak_Test_App/scripts/flatpak-pip-generator b/Flatpak_Test_App/scripts/flatpak-pip-generator new file mode 100755 index 000000000..1b58d9de2 --- /dev/null +++ b/Flatpak_Test_App/scripts/flatpak-pip-generator @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 + +__license__ = 'MIT' + +import argparse +import json +import hashlib +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request + +from collections import OrderedDict +from typing import Dict + +try: + import requirements +except ImportError: + exit('Requirements modules is not installed. Run "pip install requirements-parser"') + +parser = argparse.ArgumentParser() +parser.add_argument('packages', nargs='*') +parser.add_argument('--python2', action='store_true', + help='Look for a Python 2 package') +parser.add_argument('--cleanup', choices=['scripts', 'all'], + help='Select what to clean up after build') +parser.add_argument('--requirements-file', '-r', + help='Specify requirements.txt file') +parser.add_argument('--build-only', action='store_const', + dest='cleanup', const='all', + help='Clean up all files after build') +parser.add_argument('--build-isolation', action='store_true', + default=False, + help=( + 'Do not disable build isolation. ' + 'Mostly useful on pip that does\'t ' + 'support the feature.' + )) +parser.add_argument('--ignore-installed', + type=lambda s: s.split(','), + default='', + help='Comma-separated list of package names for which pip ' + 'should ignore already installed packages. Useful when ' + 'the package is installed in the SDK but not in the ' + 'runtime.') +parser.add_argument('--checker-data', action='store_true', + help='Include x-checker-data in output for the "Flatpak External Data Checker"') +parser.add_argument('--output', '-o', + help='Specify output file name') +parser.add_argument('--runtime', + help='Specify a flatpak to run pip inside of a sandbox, ensures python version compatibility') +parser.add_argument('--yaml', action='store_true', + help='Use YAML as output format instead of JSON') +opts = parser.parse_args() + +if opts.yaml: + try: + import yaml + except ImportError: + exit('PyYAML modules is not installed. Run "pip install PyYAML"') + + +def get_pypi_url(name: str, filename: str) -> str: + url = 'https://pypi.org/pypi/{}/json'.format(name) + print('Extracting download url for', name) + with urllib.request.urlopen(url) as response: + body = json.loads(response.read().decode('utf-8')) + for release in body['releases'].values(): + for source in release: + if source['filename'] == filename: + return source['url'] + raise Exception('Failed to extract url from {}'.format(url)) + + +def get_tar_package_url_pypi(name: str, version: str) -> str: + url = 'https://pypi.org/pypi/{}/{}/json'.format(name, version) + with urllib.request.urlopen(url) as response: + body = json.loads(response.read().decode('utf-8')) + for ext in ['bz2', 'gz', 'xz', 'zip']: + for source in body['urls']: + if source['url'].endswith(ext): + return source['url'] + err = 'Failed to get {}-{} source from {}'.format(name, version, url) + raise Exception(err) + + +def get_package_name(filename: str) -> str: + if filename.endswith(('bz2', 'gz', 'xz', 'zip')): + segments = filename.split('-') + if len(segments) == 2: + return segments[0] + return '-'.join(segments[:len(segments) - 1]) + elif filename.endswith('whl'): + segments = filename.split('-') + if len(segments) == 5: + return segments[0] + candidate = segments[:len(segments) - 4] + # Some packages list the version number twice + # e.g. PyQt5-5.15.0-5.15.0-cp35.cp36.cp37.cp38-abi3-manylinux2014_x86_64.whl + if candidate[-1] == segments[len(segments) - 4]: + return '-'.join(candidate[:-1]) + return '-'.join(candidate) + else: + raise Exception( + 'Downloaded filename: {} does not end with bz2, gz, xz, zip, or whl'.format(filename) + ) + + +def get_file_version(filename: str) -> str: + name = get_package_name(filename) + segments = filename.split(name + '-') + version = segments[1].split('-')[0] + for ext in ['tar.gz', 'whl', 'tar.xz', 'tar.gz', 'tar.bz2', 'zip']: + version = version.replace('.' + ext, '') + return version + + +def get_file_hash(filename: str) -> str: + sha = hashlib.sha256() + print('Generating hash for', filename.split('/')[-1]) + with open(filename, 'rb') as f: + while True: + data = f.read(1024 * 1024 * 32) + if not data: + break + sha.update(data) + return sha.hexdigest() + + +def download_tar_pypi(url: str, tempdir: str) -> None: + with urllib.request.urlopen(url) as response: + file_path = os.path.join(tempdir, url.split('/')[-1]) + with open(file_path, 'x+b') as tar_file: + shutil.copyfileobj(response, tar_file) + + +def parse_continuation_lines(fin): + for line in fin: + line = line.rstrip('\n') + while line.endswith('\\'): + try: + line = line[:-1] + next(fin).rstrip('\n') + except StopIteration: + exit('Requirements have a wrong number of line continuation characters "\\"') + yield line + + +def fprint(string: str) -> None: + separator = '=' * 72 # Same as `flatpak-builder` + print(separator) + print(string) + print(separator) + + +packages = [] +if opts.requirements_file: + requirements_file = os.path.expanduser(opts.requirements_file) + try: + with open(requirements_file, 'r') as req_file: + reqs = parse_continuation_lines(req_file) + reqs_as_str = '\n'.join([r.split('--hash')[0] for r in reqs]) + packages = list(requirements.parse(reqs_as_str)) + except FileNotFoundError: + pass + +elif opts.packages: + packages = list(requirements.parse('\n'.join(opts.packages))) + with tempfile.NamedTemporaryFile('w', delete=False, prefix='requirements.') as req_file: + req_file.write('\n'.join(opts.packages)) + requirements_file = req_file.name +else: + exit('Please specifiy either packages or requirements file argument') + +for i in packages: + if i["name"].lower().startswith("pyqt"): + print("PyQt packages are not supported by flapak-pip-generator") + print("However, there is a BaseApp for PyQt available, that you should use") + print("Visit https://github.com/flathub/com.riverbankcomputing.PyQt.BaseApp for more information") + sys.exit(0) + +with open(requirements_file, 'r') as req_file: + use_hash = '--hash=' in req_file.read() + +python_version = '2' if opts.python2 else '3' +if opts.python2: + pip_executable = 'pip2' +else: + pip_executable = 'pip3' + +if opts.runtime: + flatpak_cmd = [ + 'flatpak', + '--devel', + '--share=network', + '--filesystem=/tmp', + '--command={}'.format(pip_executable), + 'run', + opts.runtime + ] + if opts.requirements_file: + requirements_file = os.path.expanduser(opts.requirements_file) + if os.path.exists(requirements_file): + prefix = os.path.realpath(requirements_file) + flag = '--filesystem={}'.format(prefix) + flatpak_cmd.insert(1,flag) +else: + flatpak_cmd = [pip_executable] + +if opts.output: + output_package = opts.output +elif opts.requirements_file: + output_package = 'python{}-{}'.format( + python_version, + os.path.basename(opts.requirements_file).replace('.txt', ''), + ) +elif len(packages) == 1: + output_package = 'python{}-{}'.format( + python_version, packages[0].name, + ) +else: + output_package = 'python{}-modules'.format(python_version) +if opts.yaml: + output_filename = output_package + '.yaml' +else: + output_filename = output_package + '.json' + +modules = [] +vcs_modules = [] +sources = {} + +tempdir_prefix = 'pip-generator-{}'.format(os.path.basename(output_package)) +with tempfile.TemporaryDirectory(prefix=tempdir_prefix) as tempdir: + pip_download = flatpak_cmd + [ + 'download', + '--exists-action=i', + '--dest', + tempdir, + '-r', + requirements_file + ] + if use_hash: + pip_download.append('--require-hashes') + + fprint('Downloading sources') + cmd = ' '.join(pip_download) + print('Running: "{}"'.format(cmd)) + try: + subprocess.run(pip_download, check=True) + except subprocess.CalledProcessError: + print('Failed to download') + print('Please fix the module manually in the generated file') + + if not opts.requirements_file: + try: + os.remove(requirements_file) + except FileNotFoundError: + pass + + fprint('Downloading arch independent packages') + for filename in os.listdir(tempdir): + if not filename.endswith(('bz2', 'any.whl', 'gz', 'xz', 'zip')): + version = get_file_version(filename) + name = get_package_name(filename) + url = get_tar_package_url_pypi(name, version) + print('Deleting', filename) + try: + os.remove(os.path.join(tempdir, filename)) + except FileNotFoundError: + pass + print('Downloading {}'.format(url)) + download_tar_pypi(url, tempdir) + + files = {get_package_name(f): [] for f in os.listdir(tempdir)} + + for filename in os.listdir(tempdir): + name = get_package_name(filename) + files[name].append(filename) + + # Delete redundant sources, for vcs sources + for name in files: + if len(files[name]) > 1: + zip_source = False + for f in files[name]: + if f.endswith('.zip'): + zip_source = True + if zip_source: + for f in files[name]: + if not f.endswith('.zip'): + try: + os.remove(os.path.join(tempdir, f)) + except FileNotFoundError: + pass + + vcs_packages = { + x.name: {'vcs': x.vcs, 'revision': x.revision, 'uri': x.uri} + for x in packages + if x.vcs + } + + fprint('Obtaining hashes and urls') + for filename in os.listdir(tempdir): + name = get_package_name(filename) + sha256 = get_file_hash(os.path.join(tempdir, filename)) + + if name in vcs_packages: + uri = vcs_packages[name]['uri'] + revision = vcs_packages[name]['revision'] + vcs = vcs_packages[name]['vcs'] + url = 'https://' + uri.split('://', 1)[1] + s = 'commit' + if vcs == 'svn': + s = 'revision' + source = OrderedDict([ + ('type', vcs), + ('url', url), + (s, revision), + ]) + is_vcs = True + else: + url = get_pypi_url(name, filename) + source = OrderedDict([ + ('type', 'file'), + ('url', url), + ('sha256', sha256)]) + if opts.checker_data: + source['x-checker-data'] = { + 'type': 'pypi', + 'name': name} + if url.endswith(".whl"): + source['x-checker-data']['packagetype'] = 'bdist_wheel' + is_vcs = False + sources[name] = {'source': source, 'vcs': is_vcs} + +# Python3 packages that come as part of org.freedesktop.Sdk. +system_packages = ['cython', 'easy_install', 'mako', 'markdown', 'meson', 'pip', 'pygments', 'setuptools', 'six', 'wheel'] + +fprint('Generating dependencies') +for package in packages: + + if package.name is None: + print('Warning: skipping invalid requirement specification {} because it is missing a name'.format(package.line), file=sys.stderr) + print('Append #egg= to the end of the requirement line to fix', file=sys.stderr) + continue + elif package.name.casefold() in system_packages: + print(f"{package.name} is in system_packages. Skipping.") + continue + + if len(package.extras) > 0: + extras = '[' + ','.join(extra for extra in package.extras) + ']' + else: + extras = '' + + version_list = [x[0] + x[1] for x in package.specs] + version = ','.join(version_list) + + if package.vcs: + revision = '' + if package.revision: + revision = '@' + package.revision + pkg = package.uri + revision + '#egg=' + package.name + else: + pkg = package.name + extras + version + + dependencies = [] + # Downloads the package again to list dependencies + + tempdir_prefix = 'pip-generator-{}'.format(package.name) + with tempfile.TemporaryDirectory(prefix='{}-{}'.format(tempdir_prefix, package.name)) as tempdir: + pip_download = flatpak_cmd + [ + 'download', + '--exists-action=i', + '--dest', + tempdir, + ] + try: + print('Generating dependencies for {}'.format(package.name)) + subprocess.run(pip_download + [pkg], check=True, stdout=subprocess.DEVNULL) + for filename in sorted(os.listdir(tempdir)): + dep_name = get_package_name(filename) + if dep_name.casefold() in system_packages: + continue + dependencies.append(dep_name) + + except subprocess.CalledProcessError: + print('Failed to download {}'.format(package.name)) + + is_vcs = True if package.vcs else False + package_sources = [] + for dependency in dependencies: + if dependency in sources: + source = sources[dependency] + elif dependency.replace('_', '-') in sources: + source = sources[dependency.replace('_', '-')] + else: + continue + + if not (not source['vcs'] or is_vcs): + continue + + package_sources.append(source['source']) + + if package.vcs: + name_for_pip = '.' + else: + name_for_pip = pkg + + module_name = 'python{}-{}'.format(python_version, package.name) + + pip_command = [ + pip_executable, + 'install', + '--verbose', + '--exists-action=i', + '--no-index', + '--find-links="file://${PWD}"', + '--prefix=${FLATPAK_DEST}', + '"{}"'.format(name_for_pip) + ] + if package.name in opts.ignore_installed: + pip_command.append('--ignore-installed') + if not opts.build_isolation: + pip_command.append('--no-build-isolation') + + module = OrderedDict([ + ('name', module_name), + ('buildsystem', 'simple'), + ('build-commands', [' '.join(pip_command)]), + ('sources', package_sources), + ]) + if opts.cleanup == 'all': + module['cleanup'] = ['*'] + elif opts.cleanup == 'scripts': + module['cleanup'] = ['/bin', '/share/man/man1'] + + if package.vcs: + vcs_modules.append(module) + else: + modules.append(module) + +modules = vcs_modules + modules +if len(modules) == 1: + pypi_module = modules[0] +else: + pypi_module = { + 'name': output_package, + 'buildsystem': 'simple', + 'build-commands': [], + 'modules': modules, + } + +print() +with open(output_filename, 'w') as output: + if opts.yaml: + class OrderedDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super(OrderedDumper, self).increase_indent(flow, False) + + def dict_representer(dumper, data): + return dumper.represent_dict(data.items()) + + OrderedDumper.add_representer(OrderedDict, dict_representer) + + output.write("# Generated with flatpak-pip-generator " + " ".join(sys.argv[1:]) + "\n") + yaml.dump(pypi_module, output, Dumper=OrderedDumper) + else: + output.write(json.dumps(pypi_module, indent=4)) + print('Output saved to {}'.format(output_filename))