[utils] Properly handle list values in update_url() #2726
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| env: | |
| all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 | |
| main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11 | |
| pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7 | |
| cpython-versions: main | |
| test-set: core | |
| on: | |
| push: | |
| inputs: | |
| cpython-versions: | |
| type: string | |
| default: all | |
| test-set: | |
| type: string | |
| default: core | |
| pull_request: | |
| inputs: | |
| cpython-versions: | |
| type: string | |
| default: main | |
| test-set: | |
| type: string | |
| default: both | |
| workflow_dispatch: | |
| inputs: | |
| cpython-versions: | |
| type: choice | |
| description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11) | |
| options: | |
| - all | |
| - main | |
| required: true | |
| default: main | |
| test-set: | |
| type: choice | |
| description: core, download | |
| options: | |
| - both | |
| - core | |
| - download | |
| required: true | |
| default: both | |
| permissions: | |
| contents: read | |
| jobs: | |
| select: | |
| name: Select tests from inputs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cpython-versions: ${{ steps.run.outputs.cpython-versions }} | |
| test-set: ${{ steps.run.outputs.test-set }} | |
| own-pip-versions: ${{ steps.run.outputs.own-pip-versions }} | |
| steps: | |
| - name: Make version array | |
| id: run | |
| run: | | |
| # Make a JSON Array from comma/space-separated string (no extra escaping) | |
| json_list() { \ | |
| ret=""; IFS="${IFS},"; set -- $*; \ | |
| for a in "$@"; do \ | |
| ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \ | |
| done; \ | |
| printf '[%s]' "$ret"; } | |
| tests="${{ inputs.test-set || env.test-set }}" | |
| [ $tests = both ] && tests="core download" | |
| printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT" | |
| versions="${{ inputs.cpython-versions || env.cpython-versions }}" | |
| if [ "$versions" = all ]; then \ | |
| versions="${{ env.all-cpython-versions }}"; else \ | |
| versions="${{ env.main-cpython-versions }}"; \ | |
| fi | |
| printf 'cpython-versions=%s\n' \ | |
| "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT" | |
| # versions with a special get-pip.py in a per-version subdirectory | |
| printf 'own-pip-versions=%s\n' \ | |
| "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT" | |
| tests: | |
| name: Run tests | |
| needs: select | |
| permissions: | |
| contents: read | |
| packages: write | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| PIP: python -m pip | |
| PIP_DISABLE_PIP_VERSION_CHECK: true | |
| PIP_NO_PYTHON_VERSION_WARNING: true | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: [ubuntu-20.04] | |
| python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }} | |
| python-impl: [cpython] | |
| ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }} | |
| run-tests-ext: [sh] | |
| include: | |
| - os: windows-2019 | |
| python-version: 3.4 | |
| python-impl: cpython | |
| ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} | |
| run-tests-ext: bat | |
| - os: windows-2019 | |
| python-version: 3.4 | |
| python-impl: cpython | |
| ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} | |
| run-tests-ext: bat | |
| # jython | |
| - os: ubuntu-20.04 | |
| python-version: 2.7 | |
| python-impl: jython | |
| ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} | |
| run-tests-ext: sh | |
| - os: ubuntu-20.04 | |
| python-version: 2.7 | |
| python-impl: jython | |
| ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} | |
| run-tests-ext: sh | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| #-------- Python 3 ----- | |
| - name: Set up supported Python ${{ matrix.python-version }} | |
| id: setup-python | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}} | |
| # wrap broken actions/setup-python@v4 | |
| uses: ytdl-org/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache-build: true | |
| allow-build: info | |
| - name: Locate supported Python ${{ matrix.python-version }} | |
| if: ${{ env.pythonLocation }} | |
| shell: bash | |
| run: | | |
| echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV" | |
| export expected="${{ steps.setup-python.outputs.python-path }}" | |
| dirname() { printf '%s\n' \ | |
| 'import os, sys' \ | |
| 'print(os.path.dirname(sys.argv[1]))' \ | |
| | ${expected} - "$1"; } | |
| expd="$(dirname "$expected")" | |
| export python="$(command -v python)" | |
| [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV" | |
| [ -x "$python" ] || printf '%s\n' \ | |
| 'import os' \ | |
| 'exp = os.environ["expected"]' \ | |
| 'python = os.environ["python"]' \ | |
| 'exps = os.path.split(exp)' \ | |
| 'if python and (os.path.dirname(python) == exp[0]):' \ | |
| ' exit(0)' \ | |
| 'exps[1] = "python" + os.path.splitext(exps[1])[1]' \ | |
| 'python = os.path.join(*exps)' \ | |
| 'try:' \ | |
| ' os.symlink(exp, python)' \ | |
| 'except AttributeError:' \ | |
| ' os.rename(exp, python)' \ | |
| | ${expected} - | |
| printf '%s\n' \ | |
| 'import sys' \ | |
| 'print(sys.path)' \ | |
| | ${expected} - | |
| #-------- Python 3.12 - | |
| - name: Set up CPython 3.12 environment | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }} | |
| shell: bash | |
| run: | | |
| PYENV_ROOT=$HOME/.local/share/pyenv | |
| echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" | |
| - name: Cache Python 3.12 | |
| id: cache312 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }} | |
| uses: actions/cache@v3 | |
| with: | |
| key: python-3.12 | |
| path: | | |
| ${{ env.PYENV_ROOT }} | |
| - name: Build and set up Python 3.12 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }} | |
| # dl and build locally | |
| shell: bash | |
| run: | | |
| # Install build environment | |
| sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ | |
| libncursesw5-dev libreadline-dev libsqlite3-dev \ | |
| libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev | |
| # Download PyEnv from its GitHub repository. | |
| export PYENV_ROOT=${{ env.PYENV_ROOT }} | |
| export PATH=$PYENV_ROOT/bin:$PATH | |
| git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" | |
| pyenv install 3.12.0b4 | |
| - name: Locate Python 3.12 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }} | |
| shell: bash | |
| run: | | |
| PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4" | |
| echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" | |
| echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" | |
| #-------- Python 2.7 -- | |
| - name: Set up Python 2.7 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }} | |
| # install 2.7 | |
| shell: bash | |
| run: | | |
| sudo apt-get install -y python2 python-is-python2 | |
| echo "PYTHONHOME=/usr" >> "$GITHUB_ENV" | |
| #-------- Python 2.6 -- | |
| - name: Set up Python 2.6 environment | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} | |
| shell: bash | |
| run: | | |
| openssl_name=openssl-1.0.2u | |
| echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV" | |
| openssl_dir=$HOME/.local/opt/$openssl_name | |
| echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV" | |
| PYENV_ROOT=$HOME/.local/share/pyenv | |
| echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" | |
| sudo apt-get install -y openssl ca-certificates | |
| - name: Cache Python 2.6 | |
| id: cache26 | |
| if: ${{ matrix.python-version == '2.6' }} | |
| uses: actions/cache@v3 | |
| with: | |
| key: python-2.6.9 | |
| path: | | |
| ${{ env.openssl_dir }} | |
| ${{ env.PYENV_ROOT }} | |
| - name: Build and set up Python 2.6 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }} | |
| # dl and build locally | |
| shell: bash | |
| run: | | |
| # Install build environment | |
| sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ | |
| libncursesw5-dev libreadline-dev libsqlite3-dev \ | |
| libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev | |
| # Download and install OpenSSL 1.0.2, back in time | |
| openssl_name=${{ env.openssl_name }} | |
| openssl_targz=${openssl_name}.tar.gz | |
| openssl_dir=${{ env.openssl_dir }} | |
| openssl_inc=$openssl_dir/include | |
| openssl_lib=$openssl_dir/lib | |
| openssl_ssl=$openssl_dir/ssl | |
| curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz | |
| tar -xf $openssl_targz | |
| ( cd $openssl_name; \ | |
| ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \ | |
| --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \ | |
| make && \ | |
| make install ) | |
| rm -rf $openssl_name | |
| rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs | |
| # Download PyEnv from its GitHub repository. | |
| export PYENV_ROOT=${{ env.PYENV_ROOT }} | |
| export PATH=$PYENV_ROOT/bin:$PATH | |
| git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" | |
| # Prevent pyenv build trying (and failing) to update pip | |
| export GET_PIP=get-pip-2.6.py | |
| echo 'import sys; sys.exit(0)' > ${GET_PIP} | |
| GET_PIP=$(realpath $GET_PIP) | |
| # Build and install Python | |
| export CFLAGS="-I$openssl_inc" | |
| export LDFLAGS="-L$openssl_lib" | |
| export LD_LIBRARY_PATH="$openssl_lib" | |
| pyenv install 2.6.9 | |
| - name: Locate Python 2.6 | |
| if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} | |
| shell: bash | |
| run: | | |
| PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9" | |
| echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" | |
| echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" | |
| #-------- Jython ------ | |
| - name: Set up Java 8 | |
| if: ${{ matrix.python-impl == 'jython' }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 8 | |
| distribution: 'zulu' | |
| - name: Setup Jython environment | |
| if: ${{ matrix.python-impl == 'jython' }} | |
| shell: bash | |
| run: | | |
| echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV" | |
| echo "PIP=pip" >> "$GITHUB_ENV" | |
| - name: Cache Jython | |
| id: cachejy | |
| if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }} | |
| uses: actions/cache@v3 | |
| with: | |
| # 2.7.3 now available, may solve SNI issue | |
| key: jython-2.7.1 | |
| path: | | |
| ${{ env.JYTHON_ROOT }} | |
| - name: Install Jython | |
| if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }} | |
| shell: bash | |
| run: | | |
| JYTHON_ROOT="${{ env.JYTHON_ROOT }}" | |
| curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar | |
| java -jar jython-installer.jar -s -d "${JYTHON_ROOT}" | |
| echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH" | |
| - name: Set up cached Jython | |
| if: ${{ steps.cachejy.outputs.cache-hit }} | |
| shell: bash | |
| run: | | |
| JYTHON_ROOT="${{ env.JYTHON_ROOT }}" | |
| echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH | |
| - name: Install supporting Python 2.7 if possible | |
| if: ${{ steps.cachejy.outputs.cache-hit }} | |
| shell: bash | |
| run: | | |
| sudo apt-get install -y python2.7 || true | |
| #-------- pip --------- | |
| - name: Set up supported Python ${{ matrix.python-version }} pip | |
| if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }} | |
| # This step may run in either Linux or Windows | |
| shell: bash | |
| run: | | |
| echo "$PATH" | |
| echo "$PYTHONHOME" | |
| # curl is available on both Windows and Linux, -L follows redirects, -O gets name | |
| python -m ensurepip || python -m pip --version || { \ | |
| get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \ | |
| curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \ | |
| python get-pip.py; } | |
| - name: Set up Python 2.6 pip | |
| if: ${{ matrix.python-version == '2.6' }} | |
| shell: bash | |
| run: | | |
| python -m pip --version || { \ | |
| curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \ | |
| curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \ | |
| python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; } | |
| # work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751 | |
| echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV" | |
| - name: Set up other Python ${{ matrix.python-version }} pip | |
| if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} | |
| shell: bash | |
| run: | | |
| python -m pip --version || { \ | |
| curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \ | |
| curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \ | |
| python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; } | |
| #-------- unittest ---- | |
| - name: Upgrade Unittest for Python 2.6 | |
| if: ${{ matrix.python-version == '2.6' }} | |
| shell: bash | |
| run: | | |
| # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) | |
| $PIP -qq show unittest2 || { \ | |
| for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \ | |
| "f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \ | |
| "c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \ | |
| "17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \ | |
| "72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \ | |
| curl -L -O "https://files.pythonhosted.org/packages/${u}"; \ | |
| $PIP install ${u##*/}; \ | |
| done; } | |
| # make tests use unittest2 | |
| for test in ./test/test_*.py ./test/helper.py; do | |
| sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test" | |
| done | |
| #-------- nose -------- | |
| - name: Install nose for Python ${{ matrix.python-version }} | |
| if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == '3.12')) }} | |
| shell: bash | |
| run: | | |
| echo "$PATH" | |
| echo "$PYTHONHOME" | |
| # Use PyNose for recent Pythons instead of Nose | |
| py3ver="${{ matrix.python-version }}" | |
| py3ver=${py3ver#3.} | |
| [ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0 | |
| [ "$py3ver" -ge 9 ] && nose=pynose || nose=nose | |
| $PIP -qq show $nose || $PIP install $nose | |
| - name: Install nose for other Python 2 | |
| if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }} | |
| shell: bash | |
| run: | | |
| # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) | |
| $PIP -qq show nose || { \ | |
| curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \ | |
| $PIP install nose-1.3.7-py2-none-any.whl; } | |
| - name: Install nose for other Python 3 | |
| if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} | |
| shell: bash | |
| run: | | |
| $PIP -qq show nose || { \ | |
| curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \ | |
| $PIP install nose-1.3.7-py3-none-any.whl; } | |
| - name: Set up nosetest test | |
| if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} | |
| shell: bash | |
| run: | | |
| # set PYTHON_VER | |
| PYTHON_VER=${{ matrix.python-version }} | |
| [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}" | |
| echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV" | |
| echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV" | |
| # define a test to validate the Python version used by nosetests | |
| printf '%s\n' \ | |
| 'from __future__ import unicode_literals' \ | |
| 'import sys, os, platform' \ | |
| 'try:' \ | |
| ' import unittest2 as unittest' \ | |
| 'except ImportError:' \ | |
| ' import unittest' \ | |
| 'class TestPython(unittest.TestCase):' \ | |
| ' def setUp(self):' \ | |
| ' self.ver = os.environ["PYTHON_VER"].split("-")' \ | |
| ' def test_python_ver(self):' \ | |
| ' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \ | |
| ' self.assertTrue(sys.version.startswith(self.ver[-1]))' \ | |
| ' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \ | |
| ' def test_python_impl(self):' \ | |
| ' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \ | |
| > test/test_python.py | |
| #-------- TESTS ------- | |
| - name: Run tests | |
| if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} | |
| continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }} | |
| env: | |
| YTDL_TEST_SET: ${{ matrix.ytdl-test-set }} | |
| run: | | |
| ./devscripts/run_tests.${{ matrix.run-tests-ext }} | |
| flake8: | |
| name: Linter | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9 | |
| - name: Install flake8 | |
| run: pip install flake8 | |
| - name: Run flake8 | |
| run: flake8 . | |