run_classifier_with_Colab.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"run_classifier_with_Colab.ipynb（副本）","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyONXrpT9CjRzE+mFbkFvXG7"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"UeYvM-IJBKfo"},"source":["install tf_metrics"]},{"cell_type":"code","metadata":{"id":"rjMIzpcjbD_l"},"source":["!pip install git+https://github.com/guillaumegenthial/tf_metrics.git"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"r6zuXCsh8HcB"},"source":["默认使用 tensorflow 2.x，转换使用 tensorflow 1.x"]},{"cell_type":"code","metadata":{"id":"gj4LU3ELwW_h"},"source":["%tensorflow_version 1.x\n","import tensorflow\n","print(tensorflow.__version__)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"396ps36XKots"},"source":["#安装相关的套件\n","!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n","!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n","!apt-get update -qq 2>&1 > /dev/null\n","!apt-get -y install -qq google-drive-ocamlfuse fuse\n","from google.colab import auth\n","auth.authenticate_user()\n","from oauth2client.client import GoogleCredentials\n","creds = GoogleCredentials.get_application_default()\n","import getpass\n","!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n","vcode = getpass.getpass()\n","!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PKAaH6W8LCvE"},"source":["#加载云端硬碟，每次加载都会更新\n","!mkdir -p drive\n","!google-drive-ocamlfuse drive  -o nonempty"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aZWNOY5eLJB-"},"source":["#进入云端存有自己文件夹\n","import os\n","os.chdir('drive/Colab Notebooks/RoBERTa_Emotion_Classification')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HVrtHoPsLfgW"},"source":["#看一下文件夹里面的内容\n","!ls"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"g-nbGjb-8D2m"},"source":["显示所有当前使用的硬件"]},{"cell_type":"code","metadata":{"id":"-9XOvEiP5lm8"},"source":["from tensorflow.python.client import device_lib \n","device_lib.list_local_devices()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OuNm5c82X21j"},"source":["#!ls /proc\n","#CPU信息\n","!cat /proc/cpuinfo"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fAkhqStn_LYB"},"source":["!nvidia-smi\n","#!nvidia-smi -l 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1niQxGtQ-dAB"},"source":["import tensorflow as tf\n","tf.test.is_gpu_available()\n","tf.test.gpu_device_name()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"UpK-6BAK834j"},"source":["run_classifier.py"]},{"cell_type":"code","metadata":{"id":"1MrBYxXEHryB"},"source":["!python run_classifier.py --task_name=mytask --do_train=true --do_eval=true --data_dir=data --vocab_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt --bert_config_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json --init_checkpoint=chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_model.ckpt --max_seq_length=150 --train_batch_size=16 --learning_rate=1e-5 --num_train_epochs=2.0 --output_dir=output"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JTe0wHuChyYo"},"source":["!python run_classifier.py --task_name=mytask --do_eval=true --data_dir=data --vocab_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt --bert_config_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json --init_checkpoint=output --max_seq_length=150 --train_batch_size=16 --learning_rate=1e-5 --num_train_epochs=2.0 --output_dir=output"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Snd5dl7imwtr"},"source":["!python run_classifier.py --task_name=mytask --do_predict=true --data_dir=data --vocab_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt --bert_config_file=chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json --init_checkpoint=output --max_seq_length=150 --output_dir=output"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GTfCf0OJ2Sgy"},"source":["import pandas as pd\n","df_data = pd.read_csv(\"output/results.csv\", encoding='GB2312', engine ='python')\n","print(df_data.head(10))"],"execution_count":null,"outputs":[]}]}