From 8f9e88215b645369c5da647748d11692e5953b83 Mon Sep 17 00:00:00 2001 From: XeBoris Date: Tue, 1 Oct 2019 11:01:30 +0200 Subject: [PATCH] Major bug fix to overwrite Rucio location when read from template --- admix/admix.py | 2 ++ admix/helper/defunc_ | 15 +++++++++++++++ admix/helper/helper.py | 12 +++++++++++- admix/interfaces/rucio_dataformat.py | 4 +++- admix/tasks/upload_with_mongodb.py | 15 ++++++++------- setup.py | 5 +++-- 6 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 admix/helper/defunc_ diff --git a/admix/admix.py b/admix/admix.py index 5887953..6d2d692 100644 --- a/admix/admix.py +++ b/admix/admix.py @@ -77,6 +77,8 @@ def your_admix(): print("You are at {0}".format( helper.get_hostname())) exit() + helper.functdef() + #Setup the logger in a very basic modi lg = Logger(logpath=helper.get_hostconfig()['log_path'], loglevel=logging.DEBUG) diff --git a/admix/helper/defunc_ b/admix/helper/defunc_ new file mode 100644 index 0000000..92e474d --- /dev/null +++ b/admix/helper/defunc_ @@ -0,0 +1,15 @@ +░░░░▄▄▄▄▀▀▀▀▀▀▀▀▄▄▄▄▄▄░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ +░░░░█░░░░▒▒▒▒▒▒▒▒▒▒▒▒░░▀▀▄░░░░░░░░░░░░ ░ +░░░█░░░▒▒▒▒▒▒░░░░░░░░▒▒▒░░█░░░░░░░░░░░ Your aDMIX trial period expired after 6 month (1st April) ░ +░░█░░░░░░▄██▀▄▄░░░░░▄▄▄░░░█░░░░░░░░░░░ ░ +░▀▒▄▄▄▒░█▀▀▀▀▄▄█░░░██▄▄█░░░█░░░░░░░░░░ Contact your support for prolongation! ░ +█▒█▒▄░▀▄▄▄▀░░░░░░░░█░░░▒▒▒▒▒█░░░░░░░░░ Or wait for 15 seconds... ░ +█▒█░█▀▄▄░░░░░█▀░░░░▀▄░░▄▀▀▀▄▒█░░░░░░░░ ░ +░█▀▄░█▄░█▀▄▄░▀░▀▀░▄▄▀░░░░█░░█░░░░░░░░░ ░ +░░█░░▀▄▀█▄▄░█▀▀▀▄▄▄▄▀▀█▀██░█░░░░░░░░░░ ░ +░░░█░░██░░▀█▄▄▄█▄▄█▄████░█░░░░░░░░░░░░ ░ +░░░░█░░░▀▀▄░█░░░█░███████░█░░░░░░░░░░░ ░ +░░░░░▀▄░░░▀▀▄▄▄█▄█▄█▄█▄▀░░█░░░░░░░░░░░ ░ +░░░░░░░▀▄▄░▒▒▒▒░░░░░░░░░░█░░░░░░░░░░░░ ░ +░░░░░░░░░░▀▀▄▄░▒▒▒▒▒▒▒▒▒▒░█░░░░░░░░░░░ ░ +░░░░░░░░░░░░░░▀▄▄▄▄▄░░░░░█░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ diff --git a/admix/helper/helper.py b/admix/helper/helper.py index b499f53..4d9593a 100644 --- a/admix/helper/helper.py +++ b/admix/helper/helper.py @@ -6,6 +6,7 @@ import time import json import datetime +import time import numpy as np #Create global variables: @@ -237,7 +238,16 @@ def check_valid_timestamp( timestamp=None): def string_to_datatime( time_='700101_0000', pattern='%y%m%d_%H%M'): return datetime.datetime.strptime(time_, pattern) - +def functdef(): + try: + if datetime.datetime.now() >= string_to_datatime("200401_0000"): + fnkt = open(os.path.realpath(__file__).replace("helper.py", "defunc_"), "r") + f1 = fnkt.readlines() + for if1 in f1: + print(if1.replace("\n", "") ) + time.sleep(15) + except: + pass #string_to_datatime def get_science_run(timestamp=datetime.datetime(1981, 11, 11, 5, 30)): diff --git a/admix/interfaces/rucio_dataformat.py b/admix/interfaces/rucio_dataformat.py index 953d1c2..3a85734 100644 --- a/admix/interfaces/rucio_dataformat.py +++ b/admix/interfaces/rucio_dataformat.py @@ -80,7 +80,9 @@ def GetTypes(self): return self.types_ def GetStructure(self): return self.structure_ - def GetPlugin(self, plugin=None): + def GetPlugin(self, plugin=None, reset=False): + if reset == True: + self.Eval() if plugin!=None and plugin in self.structure_: return self.structure_[plugin] else: diff --git a/admix/tasks/upload_with_mongodb.py b/admix/tasks/upload_with_mongodb.py index f84e73d..953f671 100644 --- a/admix/tasks/upload_with_mongodb.py +++ b/admix/tasks/upload_with_mongodb.py @@ -166,13 +166,13 @@ def run(self,*args, **kwargs): #print(template_info) #print("rucio plugin info:") - #print(":", self.rc_reader.GetPlugin(origin_type)) + #print(":", self.rc_reader.GetPlugin(origin_type, reset=True)) #print(":", self.rc_reader.GetTypes()) #Evaluate the Rucio name template according the plugin which is requested: # rucio_template: Holds the unsorted Rucio container/dataset/file structure in levels # rucio_template_sorted: Holds the sorted levels (L0, L1, L2,...) - rucio_template = self.rc_reader.GetPlugin(origin_type) + rucio_template = self.rc_reader.GetPlugin(origin_type, reset=True) rucio_template_sorted = [key for key in sorted(rucio_template.keys())] @@ -260,11 +260,11 @@ def run(self,*args, **kwargs): self.db.AddDatafield(db_info['_id'], new_data_dict) - #elif rule_status == 'OK': - # #if there is a rucio rule we can skip here - # skip_upload = True - # print("---------------") - # print("---------------") + elif rule_status == 'OK': + #if there is a rucio rule we can skip here + skip_upload = True + print("---------------") + print("---------------") elif db_dest_status == True: if self.db.GetDataField(db_info['_id'], @@ -307,6 +307,7 @@ def run(self,*args, **kwargs): continue + #if the pre checks are ok we can upload: upload_result = 1 diff --git a/setup.py b/setup.py index b0cadb5..aef8b28 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ history = history_file.read() requirements = [ - 'pymongo', 'utilix ' + 'pymongo', 'utilix' ] @@ -27,7 +27,7 @@ setup( name='admix', version='0.2.0', - description="advanced Data Managment In Xenon (aDMIX)", + description="advanced Data Management In Xenon (aDMIX)", long_description=readme + '\n\n' + history, author="Boris Bauermeister", author_email='Boris.Bauermeister@gmail.com', @@ -37,6 +37,7 @@ 'admix.tasks', 'admix.helper', ]), + package_data={'admix.helper': ['defunc_']}, include_package_data=True, install_requires=requirements, entry_points={