Skip to content

Commit

Permalink
fix different lint pylint/bandit
Browse files Browse the repository at this point in the history
  • Loading branch information
noisecode3 committed Aug 15, 2024
1 parent 36a19f1 commit 496fe75
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 60 deletions.
53 changes: 45 additions & 8 deletions .neovim.lua
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ Plug('neovim/nvim-lspconfig')
Plug('williamboman/mason.nvim')
Plug('williamboman/mason-lspconfig.nvim')
Plug('mfussenegger/nvim-dap')
Plug('mfussenegger/nvim-dap-python')
Plug('nvim-neotest/nvim-nio')
Plug('rcarriga/nvim-dap-ui')

Expand All @@ -15,24 +16,60 @@ vim.api.nvim_set_keymap('n', 'ø', ":lua require'dapui'.toggle()<CR>", { noremap
-- See
-- https://sourceware.org/gdb/current/onlinedocs/gdb.html/Interpreters.html
-- https://sourceware.org/gdb/current/onlinedocs/gdb.html/Debugger-Adapter-Protocol.html
require("dap").adapters.gdb = {
id = 'gdb',
type = 'executable',
command = 'gdb',
args = { '--quiet', '--interpreter=dap' },
-- Require DAP and DAP-Python
local dap = require("dap")
require("dap-python").setup("python")

-- Python configuration
dap.configurations.python = {
{
-- The most basic Python configuration
name = "Launch file",
type = "python",
request = "launch",
program = "${file}", -- This will launch the current file
pythonPath = function()
local venv_path = vim.fn.getenv("VIRTUAL_ENV")
if venv_path and venv_path ~= vim.NIL and venv_path ~= '' then
return venv_path .. "/bin/python"
else
return "/usr/bin/python3" -- Fallback to system python
end
end,
args = function()
local input = vim.fn.input("Arguments: ")
return vim.split(input, " ", true)
end,
},
}

-- C++ configuration with GDB
dap.adapters.gdb = {
type = "executable",
command = "gdb", -- Ensure GDB is installed and available in your PATH
args = { "-i", "dap" }
}

require("dap").configurations.cpp = {
dap.configurations.cpp = {
{
name = 'Run executable (GDB)',
name = 'Launch executable (GDB)',
type = 'gdb',
request = 'launch',
program = '~/TombRaiderLinuxLauncher/TombRaiderLinuxLauncher',
stopOnEntry = false, -- or true if you want to stop at the beginning
cwd = '${workspaceFolder}',
setupCommands = {
{
text = '-enable-pretty-printing', -- Enable pretty-printing for gdb
description = 'enable pretty printing',
ignoreFailures = false
},
},
args = {}, -- Pass arguments to the executable
environment = {}, -- Set environment variables here
}
}


require("dapui").setup()

local nvim_lsp = require('lspconfig')
Expand Down
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ If it don't work try installing the
python3 module from the error, with pip, you'll need, beautifulsoup4 and tqdm

```shell
python3 getData.py https://www.trle.net/sc/levelfeatures.php?lid=3379
python3 get_data.py https://www.trle.net/sc/levelfeatures.php?lid=3379

```

Expand All @@ -112,8 +112,7 @@ This means that you have to open the data.json file and add those values
```

```shell
python3 getData.py https://www.trle.net/sc/levelfeatures.php?lid=3379
python3 addData.py data.json
python3 add_data.py data.json

```

Expand Down
File renamed without changes.
94 changes: 45 additions & 49 deletions utils/getData.py → utils/get_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,95 +7,88 @@
import json
import fcntl
import hashlib
import requests
import logging
from urllib.parse import urlparse
import requests
from bs4 import BeautifulSoup
from tqdm import tqdm
from urllib.parse import urlparse

# Set up logging
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s:%(message)s')
logging.getLogger("requests").setLevel(logging.DEBUG)

def validate_url(url):
# Kontrollera att URL:en har rätt format
parsed_url = urlparse(url)
def validate_url(url_input):
"""Set some constraints to the URL"""
parsed_url = urlparse(url_input)

# Kontrollera att URL:en har ett giltigt schema och nätverksplats (domän)
if not all([parsed_url.scheme, parsed_url.netloc]):
logging.error("Invalid URL format.")
sys.exit(1)

# Kontrollera att protokollet är https
if parsed_url.scheme != "https":
logging.error("Only HTTPS URLs are allowed.")
sys.exit(1)

# Kontrollera att domänen är trle.net eller subdomäner av trle.net
if not parsed_url.netloc.endswith("trle.net"):
logging.error("URL must belong to the domain 'trle.net'.")
sys.exit(1)

# Om alla kontroller passerar, returnera True (eller inget om du bara vill validera)
return True

def calculate_md5(url, cert):
try:
# Stream the response to handle large files
response = requests.get(url, verify=cert, stream=True, timeout=10)
response.raise_for_status()

# Get the total length of the file for the progress bar
total_length = int(response.headers.get('content-length', 0))

# Initialize the MD5 hash object
md5_hash = hashlib.md5()
def calculate_md5(url_input, cert_path_input):
"""Record md5 from download URL"""
try:
zip_response = requests.get(url_input, verify=cert_path_input, stream=True, timeout=10)
zip_response.raise_for_status()
total_length = int(zip_response.headers.get('content-length', 0))

# Initialize the progress bar
with tqdm(total=total_length, unit='B', unit_scale=True, desc="Calculating MD5") as progress_bar:
for chunk in response.iter_content(chunk_size=4096):
with tqdm(total=total_length, unit='B', unit_scale=True, desc="Calculating MD5") \
as progress_bar:
for chunk in zip_response.iter_content(chunk_size=4096):
if chunk: # filter out keep-alive new chunks
md5_hash.update(chunk)
hashlib.md5().update(chunk)
progress_bar.update(len(chunk))

# Return the hex digest of the MD5 hash
return md5_hash.hexdigest()
except requests.exceptions.RequestException as e:
logging.error(f"Failed to download {url}: {e}")
return hashlib.md5().hexdigest()
except requests.exceptions.RequestException as zip_response_error:
logging.error("Failed to download %s: %s", url_input, zip_response_error)
return None

url = None

LOCK_FILE = '/tmp/TRLE.lock'
CERT = '/etc/ssl/certs/ca-certificates.crt'
URL = None

if __name__ == "__main__":
if len(sys.argv) != 2:
logging.error("Usage: python3 getData.py URL")
sys.exit(1)
else:
url = sys.argv[1]
validate_url(url)
URL = sys.argv[1]
validate_url(URL)

lock_file = '/tmp/TRLE.lock'
try:
lock_fd = open(lock_file, 'w')
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
with open(LOCK_FILE, 'w') as lock_fd:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
logging.error("Another instance is already running")
sys.exit(1)

time.sleep(2)

cert = ('/etc/ssl/certs/ca-certificates.crt')

try:
response = requests.get(url, verify=cert, timeout=5)
response.raise_for_status()
except requests.exceptions.RequestException as e:
logging.error(f"Error fetching URL {url}: {e}")
level_response = requests.get(URL, verify=CERT, timeout=5)
level_response.raise_for_status()
except requests.exceptions.RequestException as level_response_error:
logging.error("Error fetching URL %s: %s", URL, level_response_error)
sys.exit(1)

if response.status_code == 200:
soup = BeautifulSoup(response.text, 'html.parser')
logging.debug(f'response.text: {response.text}')
if level_response.status_code == 200:
soup = BeautifulSoup(level_response.text, 'html.parser')
logging.debug('level_response.text: %s', level_response.text)

title_span = soup.find('span', class_='subHeader')
if title_span:
Expand All @@ -107,9 +100,10 @@ def calculate_md5(url, cert):
title = "missing"

author = soup.find('a', class_='linkl').get_text(strip=True) or "missing"
type = soup.find('td', string='file type:').find_next('td').get_text(strip=True) or "missing"
type_ = soup.find('td', string='file type:').find_next('td').get_text(strip=True) or "missing"
class_ = soup.find('td', string='class:').find_next('td').get_text(strip=True) or "missing"
releaseDate = soup.find('td', string='release date:').find_next('td').get_text(strip=True) or "missing"
releaseDate = soup.find('td', string='release date:').find_next('td').get_text(strip=True) \
or "missing"
difficulty_td = soup.find('td', string='difficulty:')
if difficulty_td:
next_td = difficulty_td.find_next('td')
Expand Down Expand Up @@ -145,7 +139,7 @@ def calculate_md5(url, cert):
url = download_link['href']
time.sleep(2)
try:
response2 = requests.head(url, verify=cert, timeout=5, allow_redirects=True)
response2 = requests.head(url, verify=CERT, timeout=5, allow_redirects=True)
response2.raise_for_status()

# Check if the content type is a zip file
Expand All @@ -154,7 +148,7 @@ def calculate_md5(url, cert):
zipFileName = download_url.split('/')[-1]

# Calculate the MD5 checksum without saving the file to disk
zipFileMd5 = calculate_md5(download_url, cert)
zipFileMd5 = calculate_md5(download_url, CERT)

if zipFileMd5:
print(f"Download URL: {download_url}")
Expand All @@ -171,18 +165,19 @@ def calculate_md5(url, cert):
except requests.exceptions.RequestException as e:
logging.error(f"Failed to retrieve file information from {url}: {e}")

walkthrough = ""
walkthrough_link = soup.find('a', string='Walkthrough')
if walkthrough_link:
url = 'https://www.trle.net/sc/' + walkthrough_link['href']
time.sleep(2)
try:
response3 = requests.get(url, verify=cert, timeout=5)
response3 = requests.get(url, verify=CERT, timeout=5)
response3.raise_for_status()
soup2 = BeautifulSoup(response3.text, 'html.parser')
iframe_tag = soup2.find('iframe')
iframe_src = iframe_tag['src']
url = "https://www.trle.net" + iframe_src
response4 = requests.get(url, verify=cert, timeout=5)
response4 = requests.get(url, verify=CERT, timeout=5)
if response4.status_code == 200:
walkthrough = response4.text
else:
Expand All @@ -200,7 +195,7 @@ def calculate_md5(url, cert):
data = {
"title": title,
"author": author,
"type": type,
"type": type_,
"class_": class_,
"releaseDate": releaseDate,
"difficulty": difficulty,
Expand All @@ -225,7 +220,8 @@ def calculate_md5(url, cert):
with open('data.json', 'w') as json_file:
json.dump(data, json_file)
else:
logging.error(f'Failed to retrieve content. Status code: {response.status_code}')
logging.error(f'Failed to retrieve content. Status code: {level_response.status_code}')

lock_fd.close()
os.remove(lock_file)
os.remove(LOCK_FILE)

File renamed without changes.
File renamed without changes.

0 comments on commit 496fe75

Please sign in to comment.