Skip to content

Commit

Permalink
[feat] Add resuming download feature for LCSC database
Browse files Browse the repository at this point in the history 
- Implemented the feature to resume downloading the LCSC database to handle interrupted downloads
- Referenced Issue Bouni#536 and completed the required implementation
  • Loading branch information
@Oliver0804
Oliver0804 committed Dec 4, 2024
1 parent e553877 commit c96b587
Showing 1 changed file with 102 additions and 69 deletions.
171 changes: 102 additions & 69 deletions library.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,16 +377,26 @@ def download(self):
self.state = LibraryState.DOWNLOAD_RUNNING
start = time.time()
wx.PostEvent(self.parent, ResetGaugeEvent())
# Download the zipped parts database

# Define basic variables
url_stub = "https://bouni.github.io/kicad-jlcpcb-tools/"
cnt_file = "chunk_num_fts5.txt"
cnt = 0
progress_file = os.path.join(self.datadir, "progress.txt")
chunk_file_stub = "parts-fts5.db.zip."
completed_chunks = set()

# Check if there is a progress file
if os.path.exists(progress_file):
with open(progress_file, "r") as f:
# Read completed chunk indices from the progress file
completed_chunks = set(int(line.strip()) for line in f.readlines())

# Get the total number of chunks to download
try:
r = requests.get(
url_stub + cnt_file, allow_redirects=True, stream=True, timeout=300
)
if r.status_code != requests.codes.ok: # pylint: disable=no-member
if r.status_code != requests.codes.ok:
wx.PostEvent(
self.parent,
MessageEvent(
Expand All @@ -398,88 +408,112 @@ def download(self):
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return

self.logger.debug(
"Parts db is split into %s parts. Proceeding to download...", r.text
)
cnt = int(r.text)
self.logger.debug("Removing any spurious old zip part files...")
for p in glob(str(Path(self.datadir) / (chunk_file_stub + "*"))):
self.logger.debug("Removing %s.", p)
os.unlink(p)
except Exception as e: # pylint: disable=broad-exception-caught
total_chunks = int(r.text)
except Exception as e:
wx.PostEvent(
self.parent,
MessageEvent(
title="Download Error",
text=f"Failed to download the JLCPCB database, {e}",
text=f"Failed to fetch database chunk count, {e}",
style="error",
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return

for i in range(cnt):
chunk_file = chunk_file_stub + f"{i+1:03}"
with open(os.path.join(self.datadir, chunk_file), "wb") as f:
try:
# Re-download incomplete or missing chunks
for i in range(total_chunks):
chunk_index = i + 1
chunk_file = chunk_file_stub + f"{chunk_index:03}"
chunk_path = os.path.join(self.datadir, chunk_file)

# Check if the chunk is logged as completed but the file might be incomplete
if chunk_index in completed_chunks:
if os.path.exists(chunk_path):
# Validate the size of the chunk file
try:
expected_size = int(
requests.head(url_stub + chunk_file, timeout=300).headers.get(
"Content-Length"
)
)
actual_size = os.path.getsize(chunk_path)
if actual_size == expected_size:
self.logger.debug(
f"Skipping already downloaded and validated chunk {chunk_index}."
)
continue
else:
self.logger.warning(
f"Chunk {chunk_index} is incomplete, re-downloading."
)
except Exception as e:
self.logger.warning(
f"Unable to validate chunk {chunk_index}, re-downloading. Error: {e}"
)
else:
self.logger.warning(
f"Chunk {chunk_index} marked as completed but file is missing, re-downloading."
)

# Download the chunk
try:
with open(chunk_path, "wb") as f:
r = requests.get(
url_stub + chunk_file,
allow_redirects=True,
stream=True,
timeout=300,
)
if r.status_code != requests.codes.ok: # pylint: disable=no-member
if r.status_code != requests.codes.ok:
wx.PostEvent(
self.parent,
MessageEvent(
title="Download Error",
text=f"Failed to download the JLCPCB database, error code {r.status_code}\n"
text=f"Failed to download chunk {chunk_index}, error code {r.status_code}\n"
+ "URL was:\n"
f"'{url_stub + chunk_file}'",
style="error",
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return

size = int(r.headers.get("Content-Length"))
self.logger.debug(
"Download parts db chunk %d of %d with a size of %.2fMB",
i + 1,
cnt,
size / 1024 / 1024,
"Downloading chunk %d/%d (%.2f MB)", chunk_index, total_chunks, size / 1024 / 1024
)
for data in r.iter_content(chunk_size=4096):
f.write(data)
progress = f.tell() / size * 100
wx.PostEvent(self.parent, UpdateGaugeEvent(value=progress))
except Exception as e: # pylint: disable= broad-exception-caught
wx.PostEvent(
self.parent,
MessageEvent(
title="Download Error",
text=f"Failed to download the JLCPCB database, {e}",
style="error",
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return
# rename existing parts-fts5.db to parts-fts5.db.bak, delete already existing bak file if neccesary
if os.path.exists(self.partsdb_file):
if os.path.exists(f"{self.partsdb_file}.bak"):
os.remove(f"{self.partsdb_file}.bak")
os.rename(self.partsdb_file, f"{self.partsdb_file}.bak")
# unzip downloaded parts.zip
self.logger.debug(f"Chunk {chunk_index} downloaded successfully.")

# Update progress file after successful download
with open(progress_file, "a") as f:
f.write(f"{chunk_index}\n")

except Exception as e:
wx.PostEvent(
self.parent,
MessageEvent(
title="Download Error",
text=f"Failed to download chunk {chunk_index}, {e}",
style="error",
),
)
self.state = LibraryState.INITIALIZED
return

# Delete progress file to indicate the download is complete
if os.path.exists(progress_file):
os.remove(progress_file)

# Combine and extract downloaded files
self.logger.debug("Combining and extracting zip part files...")
try:
unzip_parts(self.datadir)
except Exception as e: # pylint: disable=broad-exception-caught
except Exception as e:
wx.PostEvent(
self.parent,
MessageEvent(
Expand All @@ -489,36 +523,35 @@ def download(self):
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return
# check if partsdb_file was successfully extracted

# Check if the database file was successfully extracted
if not os.path.exists(self.partsdb_file):
if os.path.exists(f"{self.partsdb_file}.bak"):
os.rename(f"{self.partsdb_file}.bak", self.partsdb_file)
wx.PostEvent(
self.parent,
MessageEvent(
title="Download Error",
text="Failed to download the JLCPCB database, db was not extracted from zip",
style="error",
),
)
self.state = LibraryState.INITIALIZED
self.create_tables(["placeholder_invalid_column_fix_errors"])
return
else:
wx.PostEvent(self.parent, ResetGaugeEvent())
end = time.time()
wx.PostEvent(self.parent, PopulateFootprintListEvent())
wx.PostEvent(
self.parent,
MessageEvent(
title="Success",
text=f"Successfully downloaded and imported the JLCPCB database in {end-start:.2f} seconds!",
style="info",
title="Download Error",
text="Failed to extract the database file from the downloaded zip.",
style="error",
),
)
self.state = LibraryState.INITIALIZED
return

wx.PostEvent(self.parent, ResetGaugeEvent())
end = time.time()
wx.PostEvent(self.parent, PopulateFootprintListEvent())
wx.PostEvent(
self.parent,
MessageEvent(
title="Success",
text=f"Successfully downloaded and imported the JLCPCB database in {end - start:.2f} seconds!",
style="info",
),
)
self.state = LibraryState.INITIALIZED



def create_tables(self, headers):
"""Create all tables."""
Expand Down

0 comments on commit c96b587

Please sign in to comment.