From bf86dd6a5be8347680fe503ba849feacf3390757 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Thu, 8 Mar 2018 13:41:24 +0530 Subject: [PATCH 1/4] Stop after downloading json. Argument added which stops if task=json. --- main.py | 10 +++++++--- retrieval/intranet.py | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 0c673c2..096e46d 100644 --- a/main.py +++ b/main.py @@ -13,8 +13,8 @@ def judge_cal(): judge.calculate_score("final_Scores.xlsx") -def intra(): - intranet.do_all(THEME, DEADLINE) +def intra( args ): + intranet.do_all(THEME, DEADLINE, args) def check_defaulters(): analyze.get_defaulters(THEME, DEADLINE) @@ -24,7 +24,7 @@ def main( args ): , format = '%(asctime)s %(name)-10s %(levelname)-5s %(message)s' ) logging.debug( 'Calling intra' ) - intra() + intra( args ) if __name__ == '__main__': import argparse @@ -46,6 +46,10 @@ def main( args ): , dest = 'loglevel', const = logging.INFO , help = 'Be chatty. Like academic canteen and SLC.' ) + parser.add_argument('--task', '-t' + , required = False, default = 'json' + , help = 'Which task: json|all' + ) class Args: pass args = Args() parser.parse_args(namespace=args) diff --git a/retrieval/intranet.py b/retrieval/intranet.py index 1b078de..7ddaa0b 100644 --- a/retrieval/intranet.py +++ b/retrieval/intranet.py @@ -31,7 +31,6 @@ from models.photography import PhotoObject as Photo from user_constants import * - def read_main_page() -> list: """ Read entire page. All currently uploaded photos will be here @@ -273,7 +272,7 @@ def create_md_file(pics) -> None: pass -def do_all(theme: str, last_day: str): +def do_all(theme: str, last_day: str, args): """ For lazy people. This function does entire process :return: Download all photos with their information and details @@ -281,6 +280,10 @@ def do_all(theme: str, last_day: str): pics = retrieve_pics(get_all_info()) pics.sort(key=lambda x: x.points, reverse=True) save_details(pics) + if args.task == 'json': + logging.warn( 'Task %s complete. Quitting' % args.task ) + return + no_of_winners = max([2, round(len(pics) / 15)]) selected = pics[:no_of_winners] last_entry_score = selected[-1].points From 040342de492a8fe8b6bf11b11e06336e1fc3455f Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Thu, 8 Mar 2018 13:42:03 +0530 Subject: [PATCH 2/4] Default task is 'all'. If not provided, the flow is as of original script. --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 096e46d..9632bc7 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,7 @@ def main( args ): , help = 'Be chatty. Like academic canteen and SLC.' ) parser.add_argument('--task', '-t' - , required = False, default = 'json' + , required = False, default = 'all' , help = 'Which task: json|all' ) class Args: pass From afa7b232474d94e035a64ccaf18cd5d1b921b308 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Thu, 8 Mar 2018 13:48:20 +0530 Subject: [PATCH 3/4] One change to make it compatible with python3.4/python3.5. --- main.py | 15 ++++++++++++--- retrieval/intranet.py | 9 +++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 9632bc7..78ddb18 100644 --- a/main.py +++ b/main.py @@ -2,16 +2,25 @@ Main script file """ -from judging import judge from retrieval import intranet, analyze + +judge_ = True +try: + from judging import judge + judge_ = False +except ImportError as e: + print( '[WARN] Failed to import judge due to %s' % e ) + print( '\t No judging today' ) + import logging THEME = "Portraits" DEADLINE = "25 January 2018" def judge_cal(): - judge.calculate_score("final_Scores.xlsx") - + global judge_ + if judge_: + judge.calculate_score("final_Scores.xlsx") def intra( args ): intranet.do_all(THEME, DEADLINE, args) diff --git a/retrieval/intranet.py b/retrieval/intranet.py index 7ddaa0b..95bc1f0 100644 --- a/retrieval/intranet.py +++ b/retrieval/intranet.py @@ -194,8 +194,13 @@ def save_details(photos: list) -> None: # information for further process. all_info = {} for photo in photos: - key = ''.join(random.choices(string.ascii_uppercase + string.digits, - k=10)) # Create some random key + + # This is python3.6 specific. Making a python3.5 compatible version. + #key = ''.join(random.choices(string.ascii_uppercase + string.digits, + # k=10)) # Create some random key + alphas = string.ascii_uppercase + string.digits + key = ''.join( [ random.choice( alphas ) for i in range(10) ] ) + # Create json entry all_info[key] = {"uid": key, "url": photo.url, From dd7e659430d17a5de896fd11cdc362ccb57ad888 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Thu, 8 Mar 2018 14:00:00 +0530 Subject: [PATCH 4/4] Intergated to Hippo. Test before merge. --- retrieval/intranet.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/retrieval/intranet.py b/retrieval/intranet.py index 95bc1f0..96d29c8 100644 --- a/retrieval/intranet.py +++ b/retrieval/intranet.py @@ -169,11 +169,17 @@ def retrieve_pics(photos: list) -> list: count_text, re.sub('[^A-Za-z0-9]+', '', p2.title)) filename = filename[ :20] + ".jpg" # If name if more than 20 characters, - # strip it and add file extension - p2.file_name = filename # Update file_name field in the PhotoObject - testfile = request.URLopener() # start downloading - testfile.retrieve(p2.photo_url, photo_store_folder + filename) # Save - file_counter += 1 + + # if for some reason file does not get downloaded, warn and continue. + # Required for Hippo integration. + try: + # strip it and add file extension + p2.file_name = filename # Update file_name field in the PhotoObject + testfile = request.URLopener() # start downloading + testfile.retrieve(p2.photo_url, photo_store_folder + filename) # Save + file_counter += 1 + except Exception as e: + logging.warn( "Failed to download %s due to %s" % (p2.photo_url,e)) return photos @@ -215,7 +221,6 @@ def save_details(photos: list) -> None: # Dump to json with open(output_file_name + ".json", "w") as f: print(json.dumps(all_info), file=f) - logging.info( 'Wrote %s.json' % output_file_name )