From 0372eb8a41c9a7a654dcd937992d96071a6b7a95 Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Tue, 7 May 2024 15:14:18 +0200 Subject: [PATCH] Move entire collection of integrations to a single invoke task (#25348) * Move whole collection to invoke task to avoid calling invoke in a loop * Simplify by merging blocks * Fixup an error that surface after reordering Usage of the special $? variable requires a previous shell invocation via ``; this was working because there was a `` call above (not the actual one we intended to check). This commit corrects that. * Fix capitalization Co-authored-by: Dustin Long * Remove logic for old manifest --------- Co-authored-by: Dustin Long --- .../datadog-agent-integrations-py2.rb | 64 ++++--------------- .../datadog-agent-integrations-py3.rb | 56 +++------------- tasks/agent.py | 61 ++++++++++++++---- 3 files changed, 72 insertions(+), 109 deletions(-) diff --git a/omnibus/config/software/datadog-agent-integrations-py2.rb b/omnibus/config/software/datadog-agent-integrations-py2.rb index 5b0c15f776f54..d654bf752b6e0 100644 --- a/omnibus/config/software/datadog-agent-integrations-py2.rb +++ b/omnibus/config/software/datadog-agent-integrations-py2.rb @@ -120,53 +120,16 @@ cached_wheels_dir = "#{wheel_build_dir}/.cached" end - checks_to_install = Array.new - - block "Collect integrations to install" do - # Go through every integration package in `integrations-core`, build and install - Dir.glob("#{project_dir}/*").each do |check_dir| - check = check_dir.split('/').last - - # do not install excluded integrations - next if !File.directory?("#{check_dir}") || excluded_folders.include?(check) - - # If there is no manifest file, then we should assume the folder does not - # contain a working check and move onto the next - manifest_file_path = "#{check_dir}/manifest.json" - - # If there is no manifest file, then we should assume the folder does not - # contain a working check and move onto the next - File.exist?(manifest_file_path) || next - - manifest = JSON.parse(File.read(manifest_file_path)) - if manifest.key?("supported_os") - manifest["supported_os"].include?(os) || next - else - if os == "mac_os" - tag = "Supported OS::macOS" - else - tag = "Supported OS::#{os.capitalize}" - end - - manifest["tile"]["classifier_tags"].include?(tag) || next - end - - File.file?("#{check_dir}/setup.py") || File.file?("#{check_dir}/pyproject.toml") || next - # Check if it supports Python 2. - support = `inv agent.check-supports-python-version #{check_dir} 2` - if support == "False" - log.info(log_key) { "Skipping '#{check}' since it does not support Python 2." } - next - end - - checks_to_install.push(check) - end - end - - installed_list = Array.new - cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '') block "Install integrations" do tasks_dir_in = windows_safe_path(Dir.pwd) + # Collect integrations to install + checks_to_install = ( + shellout! "inv agent.collect-integrations #{project_dir} 2 #{os} #{excluded_folders.join(',')}", + :cwd => tasks_dir_in + ).stdout.split() + + # Retrieving integrations from cache + cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '') cache_branch = (shellout! "inv release.get-release-json-value base_branch", cwd: File.expand_path('..', tasks_dir_in)).stdout.strip # On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one awscli = if windows_target? then '"c:\Program files\python311\scripts\aws"' else 'aws' end @@ -186,18 +149,15 @@ shellout! "#{python} -m pip install --no-deps --no-index " \ "--find-links #{windows_safe_path(cached_wheels_dir)} -r #{windows_safe_path(cached_wheels_dir)}\\found.txt" else - shellout! "#{pip} install --no-deps --no-index " \ - " --find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt" + shellout! "#{python} -m pip install --no-deps --no-index " \ + "--find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt" end end # get list of integration wheels already installed from cache + installed_list = Array.new if cache_bucket != '' - if windows_target? - installed_out = (shellout! "#{python} -m pip list --format json").stdout - else - installed_out = (shellout! "#{pip} list --format json").stdout - end + installed_out = `#{python} -m pip list --format json` if $?.exitstatus == 0 installed = JSON.parse(installed_out) installed.each do |package| diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb index e3e6a7607aa42..1ceb159dfc1fa 100644 --- a/omnibus/config/software/datadog-agent-integrations-py3.rb +++ b/omnibus/config/software/datadog-agent-integrations-py3.rb @@ -121,53 +121,16 @@ cached_wheels_dir = "#{wheel_build_dir}/.cached" end - checks_to_install = Array.new - - block "Collect integrations to install" do - # Go through every integration package in `integrations-core`, build and install - Dir.glob("#{project_dir}/*").each do |check_dir| - check = check_dir.split('/').last - - # do not install excluded integrations - next if !File.directory?("#{check_dir}") || excluded_folders.include?(check) - - # If there is no manifest file, then we should assume the folder does not - # contain a working check and move onto the next - manifest_file_path = "#{check_dir}/manifest.json" - - # If there is no manifest file, then we should assume the folder does not - # contain a working check and move onto the next - File.exist?(manifest_file_path) || next - - manifest = JSON.parse(File.read(manifest_file_path)) - if manifest.key?("supported_os") - manifest["supported_os"].include?(os) || next - else - if os == "mac_os" - tag = "Supported OS::macOS" - else - tag = "Supported OS::#{os.capitalize}" - end - - manifest["tile"]["classifier_tags"].include?(tag) || next - end - - File.file?("#{check_dir}/setup.py") || File.file?("#{check_dir}/pyproject.toml") || next - # Check if it supports Python 3. - support = `inv agent.check-supports-python-version #{check_dir} 3` - if support == "False" - log.info(log_key) { "Skipping '#{check}' since it does not support Python 3." } - next - end - - checks_to_install.push(check) - end - end - - installed_list = Array.new - cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '') block "Install integrations" do tasks_dir_in = windows_safe_path(Dir.pwd) + # Collect integrations to install + checks_to_install = ( + shellout! "inv agent.collect-integrations #{project_dir} 3 #{os} #{excluded_folders.join(',')}", + :cwd => tasks_dir_in + ).stdout.split() + + # Retrieving integrations from cache + cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '') cache_branch = (shellout! "inv release.get-release-json-value base_branch", cwd: File.expand_path('..', tasks_dir_in)).stdout.strip # On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one awscli = if windows_target? then '"c:\Program files\python311\scripts\aws"' else 'aws' end @@ -193,8 +156,9 @@ end # get list of integration wheels already installed from cache + installed_list = Array.new if cache_bucket != '' - installed_out = (shellout! "#{python} -m pip list --format json").stdout + installed_out = `#{python} -m pip list --format json` if $?.exitstatus == 0 installed = JSON.parse(installed_out) installed.each do |package| diff --git a/tasks/agent.py b/tasks/agent.py index d22965ea6934b..86306382ff042 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -621,8 +621,7 @@ def _linux_integration_tests(ctx, race=False, remote_docker=False, go_mod="mod", ctx.run(f"{go_cmd} {prefix}") -@task -def check_supports_python_version(_, check_dir, python): +def check_supports_python_version(check_dir, python): """ Check if a Python project states support for a given major Python version. """ @@ -640,17 +639,15 @@ def check_supports_python_version(_, check_dir, python): project_metadata = data['project'] if 'requires-python' not in project_metadata: - print('True', end='') - return + return True specifier = SpecifierSet(project_metadata['requires-python']) # It might be e.g. `>=3.8` which would not immediatelly contain `3` for minor_version in range(100): if specifier.contains(f'{python}.{minor_version}'): - print('True', end='') - return + return True else: - print('False', end='') + return False elif os.path.isfile(setup_file): with open(setup_file) as f: tree = ast.parse(f.read(), filename=setup_file) @@ -659,13 +656,55 @@ def check_supports_python_version(_, check_dir, python): for node in ast.walk(tree): if isinstance(node, ast.keyword) and node.arg == 'classifiers': classifiers = ast.literal_eval(node.value) - print(any(cls.startswith(prefix) for cls in classifiers), end='') - return + return any(cls.startswith(prefix) for cls in classifiers) else: - print('False', end='') + return False else: - raise Exit('not a Python project', code=1) + return False + + +@task +def collect_integrations(_, integrations_dir, python_version, target_os, excluded): + """ + Collect and print the list of integrations to install. + + `excluded` is a comma-separated list of directories that don't contain an actual integration + """ + import json + + excluded = excluded.split(',') + integrations = [] + + for entry in os.listdir(integrations_dir): + int_path = os.path.join(integrations_dir, entry) + if not os.path.isdir(int_path) or entry in excluded: + continue + + manifest_file_path = os.path.join(int_path, "manifest.json") + + # If there is no manifest file, then we should assume the folder does not + # contain a working check and move onto the next + if not os.path.exists(manifest_file_path): + continue + + with open(manifest_file_path) as f: + manifest = json.load(f) + + # Figure out whether the integration is supported on the target OS + if target_os == 'mac_os': + tag = 'Supported OS::macOS' + else: + tag = f'Supported OS::{target_os.capitalize()}' + + if tag not in manifest['tile']['classifier_tags']: + continue + + if not check_supports_python_version(int_path, python_version): + continue + + integrations.append(entry) + print(' '.join(sorted(integrations))) @task def clean(ctx):