Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update code-style rules #215

Merged
merged 24 commits into from
Apr 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
prune .github
prune ci
prune data
prune docs
prune tutorials
prune ci
prune disdrodb/tests
8 changes: 3 additions & 5 deletions disdrodb/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import contextlib
import os
from importlib.metadata import PackageNotFoundError, version

Expand Down Expand Up @@ -33,11 +34,8 @@
"download_station",
]

__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # noqa
__root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

# Get version
try:
with contextlib.suppress(PackageNotFoundError):
__version__ = version("disdrodb")
except PackageNotFoundError:
# package is not installed
pass
18 changes: 12 additions & 6 deletions disdrodb/api/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,7 @@ def check_url(url: str) -> bool:

if re.match(regex, url):
return True
else:
return False
return False


def check_path_is_a_directory(dir_path, path_name=""):
Expand Down Expand Up @@ -130,7 +129,7 @@ def check_sensor_name(sensor_name: str, product: str = "L0A") -> None:


def check_campaign_name(campaign_name):
"""Check the campaign name is upper case !"""
"""Check the campaign name is upper case !."""
upper_campaign_name = campaign_name.upper()
if campaign_name != upper_campaign_name:
msg = f"The campaign directory name {campaign_name} must be defined uppercase: {upper_campaign_name}"
Expand All @@ -139,7 +138,7 @@ def check_campaign_name(campaign_name):


def check_data_source(data_source):
"""Check the data_source name is upper case !"""
"""Check the data_source name is upper case !."""
upper_data_source = data_source.upper()
if data_source != upper_data_source:
msg = f"The data source directory name {data_source} must be defined uppercase: {upper_data_source}"
Expand Down Expand Up @@ -207,7 +206,11 @@ def check_station_has_data(product, data_source, campaign_name, station_name, ba
def check_metadata_dir(product, data_source, campaign_name, base_dir=None):
"""Check existence of the metadata directory. If does not exists, raise an error."""
metadata_dir = define_metadata_dir(
product=product, base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
product=product,
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(metadata_dir) and os.path.isdir(metadata_dir):
msg = f"The metadata directory does not exist at {metadata_dir}."
Expand Down Expand Up @@ -253,7 +256,10 @@ def check_metadata_file(product, data_source, campaign_name, station_name, base_
def check_issue_dir(data_source, campaign_name, base_dir=None):
"""Check existence of the issue directory. If does not exists, raise an error."""
issue_dir = define_issue_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, check_exists=False
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
check_exists=False,
)
if not os.path.exists(issue_dir) and os.path.isdir(issue_dir):
msg = "The issue directory does not exist at {issue_dir}."
Expand Down
34 changes: 26 additions & 8 deletions disdrodb/api/create_directories.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import logging
import os
import shutil
from typing import Optional
ghiggi marked this conversation as resolved.
Show resolved Hide resolved

from disdrodb.api.checks import (
check_metadata_file,
Expand Down Expand Up @@ -117,7 +118,11 @@ def _check_campaign_name_consistency(raw_dir: str, processed_dir: str) -> str:


def _copy_station_metadata(
data_source: str, campaign_name: str, station_name: str, base_dir: str = None, check_validity: bool = False
data_source: str,
campaign_name: str,
station_name: str,
base_dir: Optional[str] = None,
check_validity: bool = False,
) -> None:
"""Copy the station YAML file from the ``raw_dir/metadata`` into ``processed_dir/metadata``.

Expand Down Expand Up @@ -155,11 +160,15 @@ def _copy_station_metadata(
src_filepath=raw_metadata_filepath,
dst_filepath=processed_metadata_filepath,
)
return None


def _check_pre_existing_station_data(
data_source: str, campaign_name: str, station_name: str, product: str, base_dir=None, force=False
data_source: str,
campaign_name: str,
station_name: str,
product: str,
base_dir=None,
force=False,
):
"""Check for pre-existing station data.

Expand Down Expand Up @@ -207,7 +216,6 @@ def create_l0_directory_structure(
station_name,
force,
product,
verbose=False,
ghiggi marked this conversation as resolved.
Show resolved Hide resolved
):
"""Create directory structure for the first L0 DISDRODB product.

Expand Down Expand Up @@ -328,7 +336,10 @@ def _create_station_directories(
station_name=station_name,
)
_ = create_metadata_directory(
base_dir=base_dir, product=product, data_source=data_source, campaign_name=campaign_name
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)

if product.upper() == "RAW":
Expand Down Expand Up @@ -365,6 +376,7 @@ def create_station_directory(base_dir, product, data_source, campaign_name, stat


def create_issue_directory(base_dir, data_source, campaign_name):
"""Create issue directory."""
issue_dir = define_issue_dir(
base_dir=base_dir,
data_source=data_source,
Expand Down Expand Up @@ -400,7 +412,7 @@ def create_initial_station_structure(
raise ValueError(
f"A metadata file already exists at {metadata_filepath}. "
"The station is already part of the DISDRODB Archive or "
"or you already initialized the directory structure for the station !"
"or you already initialized the directory structure for the station !",
)

# Create directory structure (/metadata, /issue and /data/<station_name>)
Expand Down Expand Up @@ -429,7 +441,10 @@ def create_initial_station_structure(

# Report location of the campaign directory
campaign_dir = define_campaign_dir(
base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, product="RAW"
base_dir=base_dir,
data_source=data_source,
campaign_name=campaign_name,
product="RAW",
)
print(f"Initial station directory structure created at: {campaign_dir}")

Expand Down Expand Up @@ -479,7 +494,10 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name,
base_dir=base_dir,
)
dst_issue_fpath = define_issue_filepath(
data_source=data_source, campaign_name=campaign_name, station_name=station_name, base_dir=test_base_dir
data_source=data_source,
campaign_name=campaign_name,
station_name=station_name,
base_dir=test_base_dir,
)
copy_file(src_issue_fpath, dst_issue_fpath)
copy_file(src_metadata_fpath, dst_metadata_fpath)
Expand Down
16 changes: 8 additions & 8 deletions disdrodb/api/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,45 +91,45 @@ def _get_version_from_filepath(filepath):


def get_version_from_filepaths(filepaths):
"""Return the DISDROB product version of the specified files"""
"""Return the DISDROB product version of the specified files."""
if isinstance(filepaths, str):
filepaths = [filepaths]
list_version = [_get_version_from_filepath(filepath) for filepath in filepaths]
return list_version


def get_campaign_name_from_filepaths(filepaths):
"""Return the DISDROB campaign name of the specified files"""
"""Return the DISDROB campaign name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="campaign_name")
return list_id


def get_station_name_from_filepaths(filepaths):
"""Return the DISDROB station name of the specified files"""
"""Return the DISDROB station name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="station_name")
return list_id


def get_product_from_filepaths(filepaths):
"""Return the DISDROB product name of the specified files"""
"""Return the DISDROB product name of the specified files."""
list_id = get_key_from_filepaths(filepaths, key="product")
return list_id


def get_start_time_from_filepaths(filepaths):
"""Return the start time of the specified files"""
"""Return the start time of the specified files."""
list_start_time = get_key_from_filepaths(filepaths, key="start_time")
return list_start_time


def get_end_time_from_filepaths(filepaths):
"""Return the end time of the specified files"""
"""Return the end time of the specified files."""
list_end_time = get_key_from_filepaths(filepaths, key="end_time")
return list_end_time


def get_start_end_time_from_filepaths(filepaths):
"""Return the start and end time of the specified files"""
"""Return the start and end time of the specified files."""
list_start_time = get_key_from_filepaths(filepaths, key="start_time")
list_end_time = get_key_from_filepaths(filepaths, key="end_time")
return np.array(list_start_time), np.array(list_end_time)
Expand Down Expand Up @@ -169,7 +169,7 @@ def infer_disdrodb_tree_path_components(path: str) -> list:
base_dir = os.path.join(*list_path_elements[: right_most_occurrence + 1])
tree_components = list_path_elements[right_most_occurrence + 1 :]
# Return components
components = [base_dir] + tree_components
components = [base_dir, *tree_components]
return components


Expand Down
60 changes: 29 additions & 31 deletions disdrodb/api/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,16 @@
def _get_campaigns_stations(base_dir, product, data_source, campaign_names):
if isinstance(campaign_names, str):
campaign_names = [campaign_names]
list_available_stations = []
for campaign_name in campaign_names:
# Get list of available stations
stations_names = _get_campaign_stations(
list_available_stations = [
(data_source, campaign_name, station_name)
for campaign_name in campaign_names
for station_name in _get_campaign_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
campaign_name=campaign_name,
)
for station_name in stations_names:
list_available_stations.append((data_source, campaign_name, station_name))

]
# Return all available stations for the asked campaigns (and specific data source)
return list_available_stations

Expand Down Expand Up @@ -135,16 +133,15 @@
def _get_data_sources_stations(base_dir, product, data_sources):
if isinstance(data_sources, str):
data_sources = [data_sources]
list_available_stations = []
for data_source in data_sources:
list_available = _get_data_source_stations(
list_available_stations = [
station_name
for data_source in data_sources
for station_name in _get_data_source_stations(
base_dir=base_dir,
product=product,
data_source=data_source,
)
for station_name in list_available:
list_available_stations.append(station_name)

]
# Return all available stations
return list_available_stations

Expand Down Expand Up @@ -195,7 +192,7 @@
idx_invalid = np.where(np.isin(data_sources, list_dir, invert=True))[0]
if len(idx_invalid) > 0:
invalid_data_sources = data_sources[idx_invalid].tolist()
raise ValueError(f"These data sources are invalid: {invalid_data_sources}.")
raise ValueError(f"These data sources does not exist: {invalid_data_sources}.")

Check warning on line 195 in disdrodb/api/io.py

View check run for this annotation

Codecov / codecov/patch

disdrodb/api/io.py#L195

Added line #L195 was not covered by tests
# Return data_sources list
data_sources = data_sources.tolist()
return data_sources
Expand Down Expand Up @@ -226,7 +223,7 @@
idx_invalid = np.where(np.isin(campaign_names, list_campaign_names, invert=True))[0]
if len(idx_invalid) > 0:
invalid_campaign_names = campaign_names[idx_invalid].tolist()
raise ValueError(f"These campaign names are invalid: {invalid_campaign_names}.")
raise ValueError(f"These campaign names does not exist: {invalid_campaign_names}.")

Check warning on line 226 in disdrodb/api/io.py

View check run for this annotation

Codecov / codecov/patch

disdrodb/api/io.py#L226

Added line #L226 was not covered by tests
# Return campaign_names list
campaign_names = campaign_names.tolist()
return campaign_names
Expand Down Expand Up @@ -270,22 +267,22 @@
campaigns = [info[1] for info in list_available_stations]
campaigns = np.unique(campaigns).tolist()
return campaigns
else:
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns
data_source_campaigns = [(info[0], info[1]) for info in list_available_stations]
data_source_campaigns = list(set(data_source_campaigns))
return data_source_campaigns

Check warning on line 272 in disdrodb/api/io.py

View check run for this annotation

Codecov / codecov/patch

disdrodb/api/io.py#L270-L272

Added lines #L270 - L272 were not covered by tests


def available_stations(
product,
data_sources=None,
campaign_names=None,
station_names=None,

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ Getting worse: Excess Number of Function Arguments
available_stations increases from 5 to 6 arguments, threshold = 4

return_tuple=True,
base_dir=None,
):
"""Return stations for which data are available on disk."""
base_dir = get_base_dir(base_dir)
# Checks
# Checks arguments
product = check_product(product)
data_sources = _check_data_sources(
base_dir=base_dir,
Expand All @@ -297,11 +294,8 @@
product=product,
campaign_names=campaign_names,
)
# Format arguments to list
if isinstance(data_sources, str):
data_sources = [data_sources]
if isinstance(campaign_names, str):
campaign_names = [campaign_names]
if isinstance(station_names, str):
station_names = [station_names]

# If data_source is None, first retrieve all stations
if data_sources is None:
Expand All @@ -313,14 +307,18 @@
data_sources=data_sources,
product=product,
)

# Then, if campaign_name is not None, subset by campaign_name
# If campaign_names is not None, subset by campaign_names
if campaign_names is not None:
list_info = [info for info in list_info if info[1] in campaign_names]

# If station_names is not None, subset by station_names
if station_names is not None:
list_info = [info for info in list_info if info[2] in station_names]

# Return list with the tuple (data_source, campaign_name, station_name)
if return_tuple:
return list_info
else:
# TODO: ENSURE THAT NO DUPLICATED STATION NAMES ?
list_stations = [info[2] for info in list_info]
return list_stations

# - Return list with the name of the available stations
list_stations = [info[2] for info in list_info]
return list_stations

Check warning on line 324 in disdrodb/api/io.py

View check run for this annotation

Codecov / codecov/patch

disdrodb/api/io.py#L323-L324

Added lines #L323 - L324 were not covered by tests
5 changes: 2 additions & 3 deletions disdrodb/api/path.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,8 @@

# Check base_dir validity
base_dir = check_base_dir(base_dir)
if len(campaign_name) > 0:
if len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")
if len(campaign_name) > 0 and len(data_source) == 0:
raise ValueError("If campaign_name is specified, data_source must be specified.")

Check warning on line 72 in disdrodb/api/path.py

View check run for this annotation

Codecov / codecov/patch

disdrodb/api/path.py#L72

Added line #L72 was not covered by tests

# Get directory
if product.upper() == "RAW":
Expand Down
Loading
Loading