Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix a couple of typos - does this help? #12

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix a couple of typos - does this help? #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 18 additions & 21 deletions FormatLog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import inspect
import gc
from datetime import datetime
#decorator functions
#decorator functions
def format_arguments(func):
""" formate many args into one string, then calls decorated function with string formatted together
""" format many args into one string, then calls decorated function with string formatted together
I.E. func("I love ",johnny.fullname,"because he is",100,"years old",**kwargs)
might look like func("I love Jonnathan because he is 100 years old",**kwargs)
"""
Expand All @@ -28,7 +28,7 @@ def format_args_and_call(self,*args,**kwargs):

def get_context_wrapper(function_to_call):
"""
passes in the context in which the wrapped function was called
passes in the context in which the wrapped function was called
"""
from functools import wraps
@wraps(function_to_call)
Expand All @@ -47,27 +47,27 @@ def context_call(self,*args,**kwargs):
funcs.append(func)
if len(funcs) > 1:
break

args.append(str(funcs[0])[10:-16] if funcs and len(funcs) == 1 else "")
return function_to_call(self,*args,**kwargs)
return context_call

class FormatLogger():
"""
singleton, in order to make everything log to the same files in the right order with out passing in the log class
singleton, in order to make everything log to the same files in the right order with out passing in the log class
"""
_instance=None
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
FormatLogger._instance.logfile = "/dev/null"
FormatLogger._instance.failure_file = "/dev/null"
FormatLogger._instance.logfile = "/dev/null"
FormatLogger._instance.failure_file = "/dev/null"
FormatLogger._instance.proccess_status = "/dev/null"
FormatLogger._instance.truncate = False
FormatLogger._instance.files = []
FormatLogger._instance.prints = 0
FormatLogger._instance.num_success = 0
FormatLogger._instance.num_fail = 0
FormatLogger._instance.truncate = False
FormatLogger._instance.files = []
FormatLogger._instance.prints = 0
FormatLogger._instance.num_success = 0
FormatLogger._instance.num_fail = 0
return cls._instance
def __init__(self):
self.truncate = self._instance.truncate
Expand Down Expand Up @@ -117,7 +117,7 @@ def write(self,desc,level = 1):
if n == level:
break
write_line_to_file(file,desc)

@format_arguments
@get_context_wrapper
def status(self,desc,cont):
Expand All @@ -143,18 +143,18 @@ def warning(self,desc,cont,context = False):
print(string)
for file in [self.logfile,self.proccess_status]:
write_line_to_file(file,string)

@format_arguments
def error(self,desc):
if self.prints <=2:
print(desc)
for file in [self.proccess_status]:
write_line_to_file(file,desc)

@format_arguments
@get_context_wrapper
def critical(self,desc,cont):# pylint: disable=E1120
string = "\n-- CRITIICAL FAILURE in {} --:{}\n".format(cont if cont else "Main Scope?",desc)
string = "\n-- CRITICAL FAILURE in {} --:{}\n".format(cont if cont else "Main Scope?",desc)
print(string)
for file in self.files:
write_line_to_file(file,string)
Expand All @@ -165,7 +165,7 @@ def success(self,desc):
print("SUCCESS: " + desc)
for file in [self.logfile,self.proccess_status]:
write_line_to_file(file,"SUCCESS: " + desc)

@format_arguments
def failure(self,desc):
self.num_fail += 1
Expand All @@ -175,7 +175,7 @@ def failure(self,desc):
write_line_to_file(file,"FAILURE: " + desc)

def close(self):
suc = "Succeded on {} out of {} total".format(self.num_success,self.num_fail+self.num_success)
suc = "Succeeded on {} out of {} total".format(self.num_success,self.num_fail+self.num_success)
if self.prints <=3:
print(suc)
write_line_to_file(self.failure_file,"Failed {} out of {} total".format(self.num_fail,self.num_fail+self.num_success))
Expand Down Expand Up @@ -211,6 +211,3 @@ def get_context():
return "- Main Scope"
elif len(stack) > 3:
return stack[3].code_context[0]



6 changes: 3 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Requires Python >= 3.5
virtualenv -p python3 ENV
source ENV/bin/activate

3. install needed python libraries
3. install needed python libraries

pip install -r requirements.txt

Expand All @@ -29,10 +29,10 @@ Requires Python >= 3.5
To run batch-loader:

`python batch_loader.py <path to csv>`
OR if instead of haveing the column `files` you have the column `fulltext_url` of the related resource
OR if instead of having the column `files` you have the column `fulltext_url` of the related resource
`python batch_loader.py <path to csv> --url`
see example.csv and url_example.csv
finally it can also be run on json files, using the same elements as the csv.
finally it can also be run on json files, using the same elements as the csv.
`python batch_loader.py <path to json file> --json`
there are more options as well such as what collections to ingest to if your
rake task can handle that, whether or not to generate tiffs, and print level.
Expand Down
6 changes: 3 additions & 3 deletions batch_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def run_ingest_process(self):
logger.failure("%s was not ingested" % (upload_id) )
self.failed.append(row)
if logger.num_success == 0 and logger.num_fail >= 5:
print("Warning: Ingest Failed frist 5 in a row!")
print("Warning: Ingest Failed first 5 in a row!")

logger.status('End of',upload_id,'\n')
except KeyboardInterrupt as yikes_stop_error:
Expand Down Expand Up @@ -172,7 +172,7 @@ def __init__(self,*args,**kwargs):
self.repeating_field_names = None # fields that will be a list
self.current = 0 # the index of the current work we going to ingest
self.base_filepath = None # where the csv is stored, used for non url ingests
self.raw_download_dir = None # temperary dircectory to download work related files
self.raw_download_dir = None # temporary directory to download work related files
self.field_names = None # original field names given in the csv

def __iter__(self):
Expand Down Expand Up @@ -378,7 +378,7 @@ class IngestFactory():
def create_controller(cls,args,config):
if args.json:#Json
ingest_controller = JsonIngestController()
else:#csv, defualt
else:#csv, default
ingest_controller = CsvIngestController()

ingest_controller.init(args.file,config.ingest_command,config.ingest_path,config.ingest_depositor,config.auth_enable,config.auth_user,config.auth_pass,args.worktype)
Expand Down
12 changes: 6 additions & 6 deletions get_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ def create_tiff_imagemagick(file):
tiff = os.path.splitext(file)[0] + '.tiff'
return_code = subprocess.run(['convert',file,tiff], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
# if return_code != 0:
# raise Exception("non zero return code for image magick convert, if you are on windows this doesnt work.\ncommand:convert {} {}".format(file,tiff))
# raise Exception("non zero return code for image magick convert, if you are on windows this does not work.\ncommand:convert {} {}".format(file,tiff))
if os.path.exists(tiff):
return tiff
logger.error('Could not create TIFF')
raise Exception("image magick convert failed to produce tiff, if you are on windows this doesnt work use magick convert instead.\n\t command: convert {} {}".format(file,tiff))
raise Exception("image magick convert failed to produce tiff, if you are on windows this does not work use magick convert instead.\n\t command: convert {} {}".format(file,tiff))

def create_dir_for(files):
"""
Expand All @@ -43,7 +43,7 @@ def create_dir_for(files):
tmpdir = tempfile.mkdtemp(dir=parentdir)
for path in files:
file_name = os.path.basename(path)
os.rename(path,os.path.join(tmpdir,file_name)) # move the file into the temporay dir basically mv(source=path,dest=tmpdir)
os.rename(path,os.path.join(tmpdir,file_name)) # move the file into the temporary dir basically mv(source=path,dest=tmpdir)
return tmpdir

def get_file_name_from_url(url):
Expand Down Expand Up @@ -83,7 +83,7 @@ def mv(path,new_path,args = None):
return subprocess.run(['sudo','mv',path,new_path]+args, stdout=subprocess.PIPE)

def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_pass=None):
""" if the given url is valid and we have access to the file attached to it. this funciton
""" if the given url is valid and we have access to the file attached to it. this function
will download said file to the directory given or just put it in the current dir.
args:
url: the url
Expand Down Expand Up @@ -168,7 +168,7 @@ def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_p
print('done downloading %s' % (local_filename),"file size:",file_size)

if file_size == 0:
logger.error("file size is 0, file must not have downlaoded correctly")
logger.error("file size is 0, file must not have downloaded correctly")
raise UrlException('Failed to downlaod')
return os.path.abspath(local_filename)
except PermissionError as e:
Expand All @@ -178,7 +178,7 @@ def download_file(url, dwnld_dir=None, auth_enable=False, auth_user=None, auth_p
if grant_access(dwnld_dir).returncode == 0:
print('success')
return download_file(url,dwnld_dir)
logger.error("could not aquire permission to download to target dir")
logger.error("could not acquire permission to download to target dir")
raise

text = ''
Expand Down