Skip to content

Commit

Permalink
Macos intel build (#2446)
Browse files Browse the repository at this point in the history
* remove conflicting dependencies

* intel preparation (untested)

* make orgName more flexible to test

* adjust set-version logic

* build-osx package target

* fix syntax

* fix syntax

* fix syntax

* append arch for upload

* syntax

* syntax

* fixes

* fix

* complete feature

* fix

* set-version now deleting the file instead of complaining if version-mismatch

* further improvements

* parametrize CI_PROJECT_ROOT_NAMESPACE with --gh-project

* more fixes

* tinyfix

* small fix

* fix packaging

* fix

* fix

* documentation and further polishing

* polishing

* bugfixes and feedback

* docs

* more polishing

* testrun

* kick

* kick

* kick again

* kick yet again

* kick

* kick again

* kick

* kick yet again

* fix

* kick

* kick again and again

* kick again

* kick

* kick again

* kick yet again

* remove comments for restriction
  • Loading branch information
k9ert authored May 29, 2024
1 parent 7fd2b91 commit 92f10e0
Show file tree
Hide file tree
Showing 17 changed files with 894 additions and 310 deletions.
79 changes: 32 additions & 47 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ release_binary_windows:
- .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG
- python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip
- cd ./pyinstaller/release
- python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
- python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
- type SHA256SUMS-windows
- echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback SHA256SUMS-windows
artifacts:
Expand Down Expand Up @@ -222,14 +222,14 @@ release_signatures:
- pip3 install -e ".[test]"
- ./utils/artifact_signer.sh init # prepare .gnupg
script:
- python3 -m utils.release-helper download # downloads the job-artifacts from gitlab
- python3 -m utils.release-helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
- python3 -m utils.release-helper checksigs # checks the signatures of all SHA256SUMM*.asc files
- python3 -m utils.release-helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
- python3 -m utils.release-helper create # creates a SHA256SUM
- python3 -m utils.release_helper download # downloads the job-artifacts from gitlab
- python3 -m utils.release_helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
- python3 -m utils.release_helper checksigs # checks the signatures of all SHA256SUMM*.asc files
- python3 -m utils.release_helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
- python3 -m utils.release_helper create # creates a SHA256SUM
- ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM
- python3 -m utils.release-helper upload_shasums # uploads SHA256SUMS to github
- python3 -m utils.release-helper upload_shasumssig # uploads SHA256SUMS.asc to github
- python3 -m utils.release_helper upload_shasums # uploads SHA256SUMS to github
- python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github

release_docker:
stage: post_releasing
Expand All @@ -247,46 +247,31 @@ tag_specterext_dummy_repo:
only:
- tags
before_script:
## Install ssh-agent if not already installed, it is required by Docker.
## (change apt-get to yum if you use an RPM-based image)
##
- 'which ssh-agent || ( apk update && apk add --no-cache bash git openssh )'
- docker info

##
## Run ssh-agent (inside the build environment)
##
- eval $(ssh-agent -s)

##
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
##
- echo "$SSH_SPECTEREXT_DEPLOY_KEY" | tr -d '\r' | ssh-add - > /dev/null

##
## Create the SSH directory and give it the right permissions
##
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh

##
## Optionally, if you will be using any Git commands, set the user name and
## and email.
##
- git config --global user.email "specter@secretvalues"
- git config --global user.name "specter"

##
## Assuming you created the SSH_KNOWN_HOSTS variable, uncomment the
## following two lines.
##
- echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts

# write access to git@github.com:cryptoadvance/specterext-dummy.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTEREXT_DEPLOY_KEY"
script:
- echo "Now tagging ... git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git"
- ./utils/tag_specterext_dummy.sh

update_github:
stage: post_releasing
only:
- tags
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/tag/${CI_COMMIT_TAG:-v2.0.4-pre8}"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate github # default-value for testing

update_webpage:
stage: post_releasing
only:
- tags
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-static.git"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate webpage # default-value for testing

55 changes: 46 additions & 9 deletions docs/release-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ Update your master branch after the release notes PR ([example](https://github.c
git tag v1.13.1 && git push upstream v1.13.1
```

if you have a proper setup of github- AND gitlab forks (and the remote `origin` on git pointing to your github fork), you can push that tag to origin and this is useful to test the procedures. See "CI/CD-dev-env setup" in [continuous-integration](./continuous-integration.md).

```bash
git tag v1.13.1 && git push origin v1.13.1
```

## GitLab - releasing stage

Creating a tag triggers the release process of the GitLab runners.
Expand Down Expand Up @@ -73,30 +79,61 @@ For details look at `.gitlab-ci.yml`

## MacOS

Ideally, directly after the tag is created, start with the MacOS release. This has to be done manually, for now. There is a script for this:
Ideally, directly after the tag is created, start with the MacOS release. As the binaries of x86/arm64 are not compatible with each other, we need to build on two MacOS architectures.This has to be done manually, for now. There is a script for this. Start with the build on x86:

### MacOS x64 build

```bash
./utils/build-osx.sh --version v2.0.5-pre4 specterd package upload
```

You can also test this procedure without messing the original project via changing the `orgName` to your `orgName` in `pyinstaller/electron/downloadloc.js`.

This will create three artifacts on github:
* specterd-v2.0.5-pre4-osx_x64.zip
* SHA256SUMS-macos_x64
* SHA256SUMS-macos_x64.asc

### MacOS arm64 build

The electron application will get built on the arm architecture. As it needs to store the sha256 hash in the electron-app, the make-hash target
will not only hash the specterd but also download the other specterd and hash it.

```bash
./utils/build-osx.sh --version v1.13.1 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" make-hash specterd electron sign upload
./utils/build-osx.sh --version v2.0.5-pre4 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" specterd make-hash electron sign package upload
```

This script also runs `github.py upload `, so two more binares and the hash and signature files are uploaded to GitHub:
This will create four artifacts on github:
* Specter-v2.0.5-pre4.dmg
* specterd-v2.0.5-pre4-osx_arm64.zip
* SHA256SUMS-macos_arm64
* SHA256SUMS-macos_arm64.asc

- Specter-v1.13.1.dmg
- specterd-v1.13.1-osx.zip
- SHA256SUMS-macos
- SHA256SUMS-macos.asc

## GitLab - post releasing

Back to GitLab, the final stage is "post releasing".

In this stage, the invididual SHA256-hashes and signatures are combined into two final files:
### release_signatures

In this job, the individual SHA256-hashes and signatures are combined into two final files:

- SHA256SUMS
- SHA256SUMS.asc

Everything, apart from the MacOS files, are pulled from the GitLab environment, the MacOS files from GitHub.
Don't forget to delete the two MacOS files (`SHA256SUMS-macos` and `SHA256SUMS-macos.asc`) on the GitHub release page in the end.
Don't forget to delete the four MacOS files (`SHA256SUMS-macos_arm64` and `SHA256SUMS-macos_arm64.asc` and the two corresponding `_x64` files) on the GitHub release page in the end.

This is difficult to automate as sometimes the manual steps has not succeeded while generating the SHASUM-files. As a result, those hashes are not included. So you might want to run this again. And you can, just delete the two generated files - `SHA256SUMS` and `SHA256SUMS.asc` and run the job again.

### release_docker

There are docker images created by the awesome [Chiang Mai LN dev](https://github.com/lncm/docker-specter-desktop). So the task of this job is to trigger their build-system which is done via `utils/trigger_docker_build.sh`. A prerequisite of this is a token in order to authenticate. That token is from Aaron, one of the maintainers of that repo, and can be found in the gitlab variables section of the CI/CD configuration.

### tag_specterext_dummy_repo

Sometimes there are changes on the plugin architecture. In order to create a plugin, it's quite important to know which version of the plugin system should be used. Because of that, we simply assume that the master of the [specterext-dummy](https://github.com/cryptoadvance/specterext-dummy) repo is compatible with the current master which was just tagged with the new version.
So this job will tag that repo with the same tag and the creation of a plugin will take the version into account.

## Trouble shooting

Expand Down
2 changes: 1 addition & 1 deletion pyinstaller/build-win-ci.bat
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pip3 install build==0.10.0
python -m build

echo " --> Installing pypi package"
python .\utils\release-helper.py install_wheel %1%
python .\utils\release_helper.py install_wheel %1%

cd pyinstaller

Expand Down
15 changes: 12 additions & 3 deletions pyinstaller/electron/downloadloc.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,23 @@

function orgName() {
// This can be changed in order to make download possible from other github orgs
return "cryptoadvance"
}

function getDownloadLocation(version, platformname) {
return `https://github.com/cryptoadvance/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
if (platformname != "osx") {
return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
}
return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}_${process.arch}.zip`
}

function appName() {
return "Specter"
}

module.exports = {
getDownloadLocation: getDownloadLocation,
appName: appName
getDownloadLocation,
appName,
orgName
}

2 changes: 1 addition & 1 deletion pyinstaller/electron/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ app.whenReady().then(() => {
if (!appSettings.versionInitialized || appSettings.versionInitialized != versionData.version) {
logger.info(`Updating ${appSettingsPath} : ${JSON.stringify(appSettings)}`)
appSettings.specterdVersion = versionData.version
appSettings.specterdHash = versionData.sha256
appSettings.specterdHash = versionData.sha256[process.arch]
appSettings.versionInitialized = versionData.version
fs.writeFileSync(appSettingsPath, JSON.stringify(appSettings))
}
Expand Down
73 changes: 54 additions & 19 deletions pyinstaller/electron/set-version.js
Original file line number Diff line number Diff line change
@@ -1,25 +1,60 @@
const fs = require('fs')
const crypto = require('crypto')
const version = process.argv[2]
const fs = require('fs');
const crypto = require('crypto');
const versionDataFile = './version-data.json';

async function setVersion() {
let package = require('./package.json')
package.version = version
fs.writeFileSync('./package.json', JSON.stringify(package, undefined, 2))

if (process.argv[3]) {
let versionData = {
version,
sha256: (await createHashFromFile(process.argv[3]))
}

fs.writeFileSync('./version-data.json', JSON.stringify(versionData, undefined, 2))
}
const version = process.argv[2];
const file = process.argv[3];
const arch = process.argv[4] || process.arch;

// Set version in package.json
let packageJson = require('./package.json');
packageJson.version = version;
fs.writeFileSync('./package.json', JSON.stringify(packageJson, undefined, 2));

// Set version in version-data.json
if (version && file) {
let versionData;
try {
versionData = require(versionDataFile);

if (versionData.version != version) {
console.log(`Version mismatch. Deleting ${versionDataFile} and creating anew.`);
fs.unlinkSync(versionDataFile); // Delete the existing version-data.json file
versionData = createNewVersionData(version); // Create new version data object
}

} catch (error) {
console.log(`No ${versionDataFile} found. Creating anew.`);
versionData = createNewVersionData(version);
}
// Compute SHA256 hash of the provided file
versionData.sha256[arch] = await createHashFromFile(file);
// Write new version data to file
fs.writeFileSync(versionDataFile, JSON.stringify(versionData, undefined, 2));
console.log("version-data.js: ")
console.log("----------------------------------------------------------")
console.log(versionData);
console.log("----------------------------------------------------------")
} else if (arch || file) {
throw new Error("Declare both arch and file or none.");
}
}

function createNewVersionData(version) {
// Return a new version data object
return {
version,
sha256: {}
};
}

const createHashFromFile = filePath => new Promise(resolve => {
const hash = crypto.createHash('sha256');
fs.createReadStream(filePath).on('data', data => hash.update(data)).on('end', () => resolve(hash.digest('hex')));
const createHashFromFile = filePath => new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
fs.createReadStream(filePath)
.on('data', data => hash.update(data))
.on('end', () => resolve(hash.digest('hex')))
.on('error', reject);
});

setVersion()
setVersion().catch(console.error);
2 changes: 1 addition & 1 deletion pyinstaller/electron/src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ function getAppSettings() {
tor: false,
proxyURL: "socks5://127.0.0.1:9050",
specterdVersion: (versionData && versionData.version !== undefined) ? versionData.version : 'unknown',
specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256 : 'unknown',
specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256[process.arch] : 'unknown',
specterdCLIArgs: "",
versionInitialized: false
}
Expand Down
2 changes: 1 addition & 1 deletion pyinstaller/electron/src/download.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ const fs = require('fs')
const { app, Menu } = require('electron')
const extract = require('extract-zip')
const { getDownloadLocation } = require('../downloadloc.js')
const { appName, appSettings, platformName, appNameLower, versionData, versionDataPath } = require('./config.js')
const { appName, appSettings, platformName, appNameLower, versionDataPath } = require('./config.js')
const { isMac, getFileHash } = require('./helpers.js')
const { logger } = require('./logging.js')
const ProgressBar = require('electron-progressbar')
Expand Down
2 changes: 0 additions & 2 deletions pyinstaller/requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,3 @@ pyinstaller==5.2
pefile==2022.5.30
macholib
pywin32-ctypes
babel==2.12.1
pytz==2022.1
8 changes: 0 additions & 8 deletions pyinstaller/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ altgraph==0.17 \
# via
# macholib
# pyinstaller
babel==2.12.1 \
--hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
--hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455
# via -r requirements.in
future==0.18.2 \
--hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
# via pefile
Expand Down Expand Up @@ -43,10 +39,6 @@ pyinstaller-hooks-contrib==2022.8 \
--hash=sha256:c4210fc50282c9c6a918e485e0bfae9405592390508e3be9fde19acc2213da56 \
--hash=sha256:e46f099934dd4577fb1ddcf37a99fa04027c92f8f5291c8802f326345988d001
# via pyinstaller
pytz==2022.1 \
--hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \
--hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c
# via -r requirements.in
pywin32-ctypes==0.2.0 \
--hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
--hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
Expand Down
Loading

0 comments on commit 92f10e0

Please sign in to comment.