Merge pull request #1803 from xlt208/lingtaox

Updating update_items.py for updating sample notebooks online

update_items.py

@@ -12,19 +12,21 @@
 
 """
 
-import os
-import sys
 import argparse
-import traceback
 import json
-import re
 import logging
-log = logging.getLogger(__name__)
+import os
+import re
+import sys
+import traceback
 
-import yaml
 import nbformat
-from nbconvert import HTMLExporter
+import requests
+import yaml
 from arcgis.gis import GIS
+from nbconvert import HTMLExporter
+
+log = logging.getLogger(__name__)
 
 ITEMS_METADATA_YAML_PATH = os.path.join(".", "items_metadata.yaml")
 THUMBNAILS_DIR = os.path.join(".", "static", "thumbnails")
@@ -35,21 +37,27 @@
 NB_PORTAL_TYPE_KEYWORDS = "Notebook, Python"
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED = \
  {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced',
- 'notebookRuntimeVersion': '5.0'}
+ 'notebookRuntimeVersion': ''}
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED_GPU = \
-  {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support',
-  'notebookRuntimeVersion': '5.0'}
+ {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support',
+ 'notebookRuntimeVersion': ''}
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD = \
  {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Standard',
- 'notebookRuntimeVersion': '5.0'}
+ 'notebookRuntimeVersion': ''}
 NB_ITEM_FOLDER = "Notebook Samples"
 
+
 def _main():
  """Parses arguments, connects to GIS, reads YAML, uploads NBs"""
  args = _parse_cmd_line_args()
  _setup_logging(args)
  gis = GIS(args.portal_url, args.username,
  args.password, verify_cert=False)
+ if args.version:
+ NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = args.version
+ NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = args.version
+ else:
+ _get_current_runtime(gis)
  items_metadata_yaml = _read_items_metadata_yaml()
  if args.replace_profiles:
  _replace_profiles()
@@ -58,100 +66,120 @@ def _main():
  if s.failed_uploads:
  raise Exception(f"Some uploads failed: {s.failed_uploads}")
 
+
 def _parse_cmd_line_args():
  """Parse CMD args, returns an object instance of all user passed in args"""
- parser = argparse.ArgumentParser(description = "Takes all notebooks "\
- "this in `gallery` directory, and will upload it to the specified "\
- "portal/org in the right group with the right categories. "\
- "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)",
- formatter_class=argparse.RawTextHelpFormatter)
+ parser = argparse.ArgumentParser(description="Takes all notebooks "
+  "this in `gallery` directory, and will upload it to the specified "
+  "portal/org in the right group with the right categories. "
+  "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)",
+  formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument("--username", "-u", type=str,
- help="Required username for the portal/org")
+  help="Required username for the portal/org")
  parser.add_argument("--password", "-p", type=str,
- help="Required password for the portal/org")
+  help="Required password for the portal/org")
  parser.add_argument("--portal-url", "-r", type=str,
- help="The portal to connect to (Default:geosaurus.maps.arcgis.com)",
- default="https://geosaurus.maps.arcgis.com/")
+ help="The portal to connect to (default: geosaurus.maps.arcgis.com)",
+ default="https://geosaurus.maps.arcgis.com/")
+ parser.add_argument("--version", "-ver", type=str,
+ help="The version of notebook runtime to set on sample notebooks "
+ "(default: the latest version)",
+ default="")
  parser.add_argument("--verbose", "-v", action="store_true",
- help="Print all DEBUG log messages instead of just INFO")
+  help="Print all DEBUG log messages instead of just INFO")
  parser.add_argument("--replace-profiles", "-c", action="store_true",
- help="Replace all profiles in notebooks with their appropriate username "\
- "and passwords. Does this by running misc/tools/replace_profiles.py")
- args = parser.parse_args(sys.argv[1:]) #don't use filename as 1st arg
+  help="Replace all profiles in notebooks with their appropriate username "
+  "and passwords. Does this by running misc/tools/replace_profiles.py")
+ args = parser.parse_args(sys.argv[1:])  # don't use filename as 1st arg
  return args
 
+
 def _setup_logging(args):
  """Sets up the logging based on args"""
  if args.verbose:
  log.setLevel(logging.DEBUG)
  else:
-  log.setLevel(logging.INFO)
+ log.setLevel(logging.INFO)
  stdout_handler = logging.StreamHandler(stream=sys.stdout)
  stdout_handler.setLevel(logging.DEBUG)
  stdout_handler.setFormatter(logging.Formatter(
- '----- %(levelname)s | '\
- '%(asctime)s | '\
- '%(filename)s line %(lineno)d'\
- ' -----\n'\
+ '----- %(levelname)s | '
+ '%(asctime)s | '
+ '%(filename)s line %(lineno)d'
+ ' -----\n'
  '"%(message)s"'))
  log.addHandler(stdout_handler)
  log.info("Logging at level {}.".format(logging.getLevelName(log.level)))
  log.debug("args passed in => {}".format(args))
 
+
+def _get_current_runtime(gis):
+ ntbk_svr = gis.notebook_server[0]
+ rest = ntbk_svr._url.replace("/admin", "/rest")
+ rest_info = requests.get(f"{rest}/info?f=json", timeout=5, verify=False)
+ version = rest_info.json()["currentRuntimeVersion"]
+ NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = version
+ NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = version
+ return version
+
+
 def _read_items_metadata_yaml():
  """Returns the items_metadata.yaml file as a dict"""
- with open(ITEMS_METADATA_YAML_PATH) as f:
+ with open(ITEMS_METADATA_YAML_PATH, encoding="utf-8") as f:
  return yaml.safe_load(f)
 
+
 def _replace_profiles():
  """Runs misc/tools/replace_profiles.py to go through each notebook in the
  repo and replace profiles with usernames/passwords
  """
  cmd = f"{sys.executable} {REPLACE_PROFILES_SCRIPT}"
  os.system(cmd)
 
+
 class ItemsUploader:
  def __init__(self, gis, items_metadata_yaml):
  self._gis = gis
  self._items_metadata_yaml = items_metadata_yaml
  self.failed_uploads = []
 
- def upload_items(self, share_after_upload = True):
+ def upload_items(self, share_after_upload=True):
  for entry in self._items_metadata_yaml["samples"] + \
-  self._items_metadata_yaml["guides"] + \
-  self._items_metadata_yaml["labs"]:
+ self._items_metadata_yaml["guides"] + \
+ self._items_metadata_yaml["labs"]:
  self._stage_and_upload_item(entry, share_after_upload)
 
- def _stage_and_upload_item(self, entry, share_after_upload = True):
+ def _stage_and_upload_item(self, entry, share_after_upload=True):
  log.info(f"Uploading {entry['title']}")
  log.debug(f" sample: {entry}")
  try:
  nb_path = entry["path"]
  self._preupload_check(entry['title'], nb_path)
- runtime_stamp = self._infer_runtime_stamp(entry.get("runtime", "standard"))
+ runtime_stamp = self._infer_runtime_stamp(
+ entry.get("runtime", "standard"))
  categories = entry.get("categories", None)
  self._stamp_file_with_runtime(nb_path, runtime_stamp)
  item_id = self._infer_item_id(entry["url"])
  item = self.update_item(
-  item_id = item_id,
-  item_type = NB_PORTAL_TYPE,
-  item_type_keywords = NB_PORTAL_TYPE_KEYWORDS,
-  title = entry['title'],
-  categories = categories,
-  snippet = entry['snippet'],
-  description = entry['description'],
-  license_info = entry['licenseInfo'],
-  tags = entry['tags'],
-  nb_path = nb_path,
-  runtime_stamp = runtime_stamp,
-  thumbnail = entry['thumbnail'])
+ item_id=item_id,
+ item_type=NB_PORTAL_TYPE,
+ item_type_keywords=NB_PORTAL_TYPE_KEYWORDS,
+ title=entry['title'],
+ categories=categories,
+ snippet=entry['snippet'],
+ description=entry['description'],
+ license_info=entry['licenseInfo'],
+ tags=entry['tags'],
+ nb_path=nb_path,
+ runtime_stamp=runtime_stamp,
+ thumbnail=entry['thumbnail'])
  if share_after_upload:
- item.share(everyone = True)
+ item.sharing.sharing_level = "EVERYONE"
  item.protect()
  if categories:
  self._assign_categories_to_item(item, categories)
  self._apply_html_preview_to_item(item, nb_path)
- log.info(f" Uploaded succeded -> {item.homepage}")
+ log.info(f" Uploaded succeeded -> {item.homepage}")
  except Exception as e:
  self.failed_uploads.append(entry['title'])
  log.warn(f" Couldn't upload {entry['title']}: {e}")
@@ -170,11 +198,11 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories,
  snippet, description, license_info, tags, nb_path,
  runtime_stamp, thumbnail):
  """Actually uploads the notebook item to the portal"""
- item_properties = {"title" : title,
- "snippet" : snippet,
- "description" : description,
- "licenseInfo" : license_info,
- "tags" : tags,
+ item_properties = {"title": title,
+ "snippet": snippet,
+ "description": description,
+ "licenseInfo": license_info,
+ "tags": tags,
  "properties": runtime_stamp}
  if categories:
  item_properties["categories"] = categories
@@ -188,11 +216,12 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories,
  log.debug(f'item {existing_item.homepage} exists, updating...')
  item_properties["url"] = existing_item.homepage
  existing_item.update(item_properties,
- data = nb_path,
- thumbnail = thumbnail)
+ data=nb_path,
+ thumbnail=thumbnail)
  resp = existing_item
  else:
- raise Exception(f"Could not find item {item_id} to update. Failing!")
+ raise Exception(
+ f"Could not find item {item_id} to update. Failing!")
  return resp
 
  def _assign_categories_to_item(self, item, categories):
@@ -207,8 +236,8 @@ def _apply_html_preview_to_item(self, item, nb_path):
 
  json_file_name = "notebook_preview.json"
  json_file_path = os.path.join(".", json_file_name)
- with open(json_file_path, 'w') as f:
- json.dump({"html" : html_str}, f)
+ with open(json_file_path, 'w', encoding="utf-8") as f:
+ json.dump({"html": html_str}, f)
 
  if item.resources.list():
  item.resources.remove()
@@ -239,11 +268,12 @@ def _stamp_file_with_runtime(self, notebook_file_path, runtime_stamp):
  nb['metadata']['esriNotebookRuntime'] = runtime_stamp
  nbformat.write(nb, notebook_file_path, nbformat.NO_CONVERT)
 
+
 if __name__ == "__main__":
  try:
  _main()
  sys.exit(0)
  except Exception as e:
  log.exception(e)
- log.info("Program did not succesfully complete (unhandled exception)")
+ log.info("Program did not successfully complete (unhandled exception)")
  sys.exit(1)