Skip to content

Commit

Permalink
Move entire collection of integrations to a single invoke task (#25348)
Browse files Browse the repository at this point in the history 
* Move whole collection to invoke task to avoid calling invoke in a loop

* Simplify by merging blocks

* Fixup an error that surface after reordering

Usage of the special $? variable requires a previous shell invocation
via ``; this was working because there was a `` call above (not the
actual one we intended to check). This commit corrects that.

* Fix capitalization

Co-authored-by: Dustin Long <me@dustmop.io>

* Remove logic for old manifest

---------

Co-authored-by: Dustin Long <me@dustmop.io>
  • Loading branch information
@alopezz @dustmop
alopezz and dustmop authored May 7, 2024
1 parent d2db304 commit 0372eb8
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 109 deletions.
64 changes: 12 additions & 52 deletions omnibus/config/software/datadog-agent-integrations-py2.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,53 +120,16 @@
cached_wheels_dir = "#{wheel_build_dir}/.cached"
end

checks_to_install = Array.new

block "Collect integrations to install" do
# Go through every integration package in `integrations-core`, build and install
Dir.glob("#{project_dir}/*").each do |check_dir|
check = check_dir.split('/').last

# do not install excluded integrations
next if !File.directory?("#{check_dir}") || excluded_folders.include?(check)

# If there is no manifest file, then we should assume the folder does not
# contain a working check and move onto the next
manifest_file_path = "#{check_dir}/manifest.json"

# If there is no manifest file, then we should assume the folder does not
# contain a working check and move onto the next
File.exist?(manifest_file_path) || next

manifest = JSON.parse(File.read(manifest_file_path))
if manifest.key?("supported_os")
manifest["supported_os"].include?(os) || next
else
if os == "mac_os"
tag = "Supported OS::macOS"
else
tag = "Supported OS::#{os.capitalize}"
end

manifest["tile"]["classifier_tags"].include?(tag) || next
end

File.file?("#{check_dir}/setup.py") || File.file?("#{check_dir}/pyproject.toml") || next
# Check if it supports Python 2.
support = `inv agent.check-supports-python-version #{check_dir} 2`
if support == "False"
log.info(log_key) { "Skipping '#{check}' since it does not support Python 2." }
next
end

checks_to_install.push(check)
end
end

installed_list = Array.new
cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
block "Install integrations" do
tasks_dir_in = windows_safe_path(Dir.pwd)
# Collect integrations to install
checks_to_install = (
shellout! "inv agent.collect-integrations #{project_dir} 2 #{os} #{excluded_folders.join(',')}",
:cwd => tasks_dir_in
).stdout.split()

# Retrieving integrations from cache
cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
cache_branch = (shellout! "inv release.get-release-json-value base_branch", cwd: File.expand_path('..', tasks_dir_in)).stdout.strip
# On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one
awscli = if windows_target? then '"c:\Program files\python311\scripts\aws"' else 'aws' end
Expand All @@ -186,18 +149,15 @@
shellout! "#{python} -m pip install --no-deps --no-index " \
"--find-links #{windows_safe_path(cached_wheels_dir)} -r #{windows_safe_path(cached_wheels_dir)}\\found.txt"
else
shellout! "#{pip} install --no-deps --no-index " \
" --find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt"
shellout! "#{python} -m pip install --no-deps --no-index " \
"--find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt"
end
end

# get list of integration wheels already installed from cache
installed_list = Array.new
if cache_bucket != ''
if windows_target?
installed_out = (shellout! "#{python} -m pip list --format json").stdout
else
installed_out = (shellout! "#{pip} list --format json").stdout
end
installed_out = `#{python} -m pip list --format json`
if $?.exitstatus == 0
installed = JSON.parse(installed_out)
installed.each do |package|
Expand Down
56 changes: 10 additions & 46 deletions omnibus/config/software/datadog-agent-integrations-py3.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,53 +121,16 @@
cached_wheels_dir = "#{wheel_build_dir}/.cached"
end

checks_to_install = Array.new

block "Collect integrations to install" do
# Go through every integration package in `integrations-core`, build and install
Dir.glob("#{project_dir}/*").each do |check_dir|
check = check_dir.split('/').last

# do not install excluded integrations
next if !File.directory?("#{check_dir}") || excluded_folders.include?(check)

# If there is no manifest file, then we should assume the folder does not
# contain a working check and move onto the next
manifest_file_path = "#{check_dir}/manifest.json"

# If there is no manifest file, then we should assume the folder does not
# contain a working check and move onto the next
File.exist?(manifest_file_path) || next

manifest = JSON.parse(File.read(manifest_file_path))
if manifest.key?("supported_os")
manifest["supported_os"].include?(os) || next
else
if os == "mac_os"
tag = "Supported OS::macOS"
else
tag = "Supported OS::#{os.capitalize}"
end

manifest["tile"]["classifier_tags"].include?(tag) || next
end

File.file?("#{check_dir}/setup.py") || File.file?("#{check_dir}/pyproject.toml") || next
# Check if it supports Python 3.
support = `inv agent.check-supports-python-version #{check_dir} 3`
if support == "False"
log.info(log_key) { "Skipping '#{check}' since it does not support Python 3." }
next
end

checks_to_install.push(check)
end
end

installed_list = Array.new
cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
block "Install integrations" do
tasks_dir_in = windows_safe_path(Dir.pwd)
# Collect integrations to install
checks_to_install = (
shellout! "inv agent.collect-integrations #{project_dir} 3 #{os} #{excluded_folders.join(',')}",
:cwd => tasks_dir_in
).stdout.split()

# Retrieving integrations from cache
cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '')
cache_branch = (shellout! "inv release.get-release-json-value base_branch", cwd: File.expand_path('..', tasks_dir_in)).stdout.strip
# On windows, `aws` actually executes Ruby's AWS SDK, but we want the Python one
awscli = if windows_target? then '"c:\Program files\python311\scripts\aws"' else 'aws' end
Expand All @@ -193,8 +156,9 @@
end

# get list of integration wheels already installed from cache
installed_list = Array.new
if cache_bucket != ''
installed_out = (shellout! "#{python} -m pip list --format json").stdout
installed_out = `#{python} -m pip list --format json`
if $?.exitstatus == 0
installed = JSON.parse(installed_out)
installed.each do |package|
Expand Down
61 changes: 50 additions & 11 deletions tasks/agent.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,8 +621,7 @@ def _linux_integration_tests(ctx, race=False, remote_docker=False, go_mod="mod",
ctx.run(f"{go_cmd} {prefix}")


@task
def check_supports_python_version(_, check_dir, python):
def check_supports_python_version(check_dir, python):
"""
Check if a Python project states support for a given major Python version.
"""
Expand All @@ -640,17 +639,15 @@ def check_supports_python_version(_, check_dir, python):

project_metadata = data['project']
if 'requires-python' not in project_metadata:
print('True', end='')
return
return True

specifier = SpecifierSet(project_metadata['requires-python'])
# It might be e.g. `>=3.8` which would not immediatelly contain `3`
for minor_version in range(100):
if specifier.contains(f'{python}.{minor_version}'):
print('True', end='')
return
return True
else:
print('False', end='')
return False
elif os.path.isfile(setup_file):
with open(setup_file) as f:
tree = ast.parse(f.read(), filename=setup_file)
Expand All @@ -659,13 +656,55 @@ def check_supports_python_version(_, check_dir, python):
for node in ast.walk(tree):
if isinstance(node, ast.keyword) and node.arg == 'classifiers':
classifiers = ast.literal_eval(node.value)
print(any(cls.startswith(prefix) for cls in classifiers), end='')
return
return any(cls.startswith(prefix) for cls in classifiers)
else:
print('False', end='')
return False
else:
raise Exit('not a Python project', code=1)
return False


@task
def collect_integrations(_, integrations_dir, python_version, target_os, excluded):
"""
Collect and print the list of integrations to install.
`excluded` is a comma-separated list of directories that don't contain an actual integration
"""
import json

excluded = excluded.split(',')
integrations = []

for entry in os.listdir(integrations_dir):
int_path = os.path.join(integrations_dir, entry)
if not os.path.isdir(int_path) or entry in excluded:
continue

manifest_file_path = os.path.join(int_path, "manifest.json")

# If there is no manifest file, then we should assume the folder does not
# contain a working check and move onto the next
if not os.path.exists(manifest_file_path):
continue

with open(manifest_file_path) as f:
manifest = json.load(f)

# Figure out whether the integration is supported on the target OS
if target_os == 'mac_os':
tag = 'Supported OS::macOS'
else:
tag = f'Supported OS::{target_os.capitalize()}'

if tag not in manifest['tile']['classifier_tags']:
continue

if not check_supports_python_version(int_path, python_version):
continue

integrations.append(entry)

print(' '.join(sorted(integrations)))

@task
def clean(ctx):
Expand Down

0 comments on commit 0372eb8

Please sign in to comment.