Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ignore_hash option in settings.ini #684

Merged
merged 11 commits into from
Nov 5, 2023
13 changes: 11 additions & 2 deletions aqt/archives.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

from defusedxml import ElementTree

from aqt.exceptions import ArchiveDownloadError, ArchiveListError, NoPackageFound
from aqt.exceptions import ArchiveDownloadError, ArchiveListError, ChecksumDownloadFailure, NoPackageFound
from aqt.helper import Settings, get_hash, getUrl, ssplit
from aqt.metadata import QtRepoProperty, Version

Expand Down Expand Up @@ -390,7 +390,16 @@ def _get_archives_base(self, name, target_packages):

def _download_update_xml(self, update_xml_path):
"""Hook for unit test."""
xml_hash = get_hash(update_xml_path, "sha256", self.timeout)
if not Settings.ignore_hash:
try:
xml_hash = get_hash(update_xml_path, Settings.hash_algorithm, self.timeout)
except ChecksumDownloadFailure:
self.logger.warning(
"Failed to download checksum for the file 'Updates.xml'. This may happen on unofficial mirrors."
)
xml_hash = None
else:
xml_hash = None
return getUrl(posixpath.join(self.base, update_xml_path), self.timeout, xml_hash)

def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages: Optional[ModuleToPackage]):
Expand Down
17 changes: 16 additions & 1 deletion aqt/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,14 @@ def getUrl(url: str, timeout: Tuple[float, float], expected_hash: Optional[bytes
raise ArchiveDownloadError(msg)
result: str = r.text
filename = url.split("/")[-1]
actual_hash = hashlib.sha256(bytes(result, "utf-8")).digest()
if Settings.hash_algorithm == "sha256":
actual_hash = hashlib.sha256(bytes(result, "utf-8")).digest()
elif Settings.hash_algorithm == "sha1":
actual_hash = hashlib.sha1(bytes(result, "utf-8")).digest()
miurahr marked this conversation as resolved.
Show resolved Hide resolved
miurahr marked this conversation as resolved.
Show resolved Hide resolved
elif Settings.hash_algorithm == "md5":
actual_hash = hashlib.md5(bytes(result, "utf-8")).digest()
miurahr marked this conversation as resolved.
Show resolved Hide resolved
miurahr marked this conversation as resolved.
Show resolved Hide resolved
else:
raise ArchiveChecksumError(f"Unknown hash algorithm: {Settings.hash_algorithm}.\nPlease check settings.ini")
if expected_hash is not None and expected_hash != actual_hash:
raise ArchiveChecksumError(
f"Downloaded file {filename} is corrupted! Detect checksum error.\n"
Expand Down Expand Up @@ -456,6 +463,14 @@ def max_retries_on_checksum_error(self):
def max_retries_to_retrieve_hash(self):
return self.config.getint("requests", "max_retries_to_retrieve_hash", fallback=int(self.max_retries))

@property
def hash_algorithm(self):
return self.config.get("requests", "hash_algorithm", fallback="sha256")

@property
def ignore_hash(self):
return self.config.getboolean("requests", "INSECURE_NOT_FOR_PRODUCTION_ignore_hash", fallback=False)

@property
def backoff_factor(self):
return self.config.getfloat("requests", "retry_backoff", fallback=0.1)
Expand Down
4 changes: 2 additions & 2 deletions aqt/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1209,12 +1209,12 @@ def installer(
logger.addHandler(qh)
#
timeout = (Settings.connection_timeout, Settings.response_timeout)
hash = get_hash(qt_package.archive_path, algorithm="sha256", timeout=timeout)
hash = get_hash(qt_package.archive_path, Settings.hash_algorithm, timeout) if not Settings.ignore_hash else None

def download_bin(_base_url):
url = posixpath.join(_base_url, qt_package.archive_path)
logger.debug("Download URL: {}".format(url))
return downloadBinaryFile(url, archive, "sha256", hash, timeout)
return downloadBinaryFile(url, archive, Settings.hash_algorithm, hash, timeout)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

12% of developers fix this issue

reportGeneralTypeIssues: Argument of type "bytes | None" cannot be assigned to parameter "exp" of type "bytes" in function "downloadBinaryFile"
  Type "bytes | None" cannot be assigned to type "bytes"
    Type "None" cannot be assigned to type "bytes"


ℹ️ Expand to see all @sonatype-lift commands

You can reply with the following commands. For example, reply with @sonatype-lift ignoreall to leave out all findings.

Command Usage
@sonatype-lift ignore Leave out the above finding from this PR
@sonatype-lift ignoreall Leave out all the existing findings from this PR
@sonatype-lift exclude <file|issue|path|tool> Exclude specified file|issue|path|tool from Lift findings by updating your config.toml file

Note: When talking to LiftBot, you need to refresh the page to see its response.
Click here to add LiftBot to another repo.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As Sonatype pointed out for you @lebarsfa , downloadBinaryFile method only accept argument passed by hash should be bytes variable type.

see def downloadBinaryFile(url: str, out: Path, hash_algo: str, exp: bytes, timeout: Tuple[float, float]) -> None:

You make variable hash able to be None

hash = get_hash(qt_package.archive_path, Settings.hash_algorithm, timeout) if not Settings.ignore_hash else None

Please fix a type inconsistency.


retry_on_errors(
action=lambda: retry_on_bad_connection(download_bin, base_url),
Expand Down
4 changes: 2 additions & 2 deletions aqt/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ def _to_version(self, qt_ver: str, arch: Optional[str]) -> Version:

def fetch_http(self, rest_of_url: str, is_check_hash: bool = True) -> str:
timeout = (Settings.connection_timeout, Settings.response_timeout)
expected_hash = get_hash(rest_of_url, "sha256", timeout) if is_check_hash else None
expected_hash = get_hash(rest_of_url, Settings.hash_algorithm, timeout) if is_check_hash else None
base_urls = self.base_url, random.choice(Settings.fallbacks)

err: BaseException = AssertionError("unraisable")
Expand Down Expand Up @@ -784,7 +784,7 @@ def _get_qt_version_str(self, version: Version) -> str:

def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[Element], bool]] = None):
rest_of_url = posixpath.join(self.archive_id.to_url(), folder, "Updates.xml")
xml = self.fetch_http(rest_of_url)
xml = self.fetch_http(rest_of_url) if not Settings.ignore_hash else self.fetch_http(rest_of_url, False)
return xml_to_modules(
xml,
predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads,
Expand Down
2 changes: 2 additions & 0 deletions aqt/settings.ini
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ max_retries_on_connection_error: 5
retry_backoff: 0.1
max_retries_on_checksum_error: 5
max_retries_to_retrieve_hash: 5
hash_algorithm: sha256
INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False

[mirrors]
trusted_mirrors:
Expand Down
13 changes: 13 additions & 0 deletions docs/configuration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ A file is like as follows:
retry_backoff: 0.1
max_retries_on_checksum_error: 5
max_retries_to_retrieve_hash: 5
hash_algorithm: sha256
INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False

[mirrors]
trusted_mirrors:
Expand Down Expand Up @@ -130,6 +132,17 @@ max_retries_on_checksum_error:
This setting controls how many times ``aqt`` will attempt to download a file,
in the case of a checksum error.

hash_algorithm:
This is either ``sha256``, ``sha1`` or ``md5``. ``sha256`` is the only safe
value to use here. See also ``trusted_mirrors`` setting.

INSECURE_NOT_FOR_PRODUCTION_ignore_hash:
This is either ``True`` or ``False``.
The ``True`` setting disables hash checking when downloading files. Although
this is not recommended, this may help when hashes are not available.
The ``False`` setting will enforce hash checking. This is highly recommended
to avoid corrupted files.


The ``[mirrors]`` section is a configuration for mirror handling.

Expand Down