added debugging and experiments

muhyun · Apr 6, 2020 · dee177f · dee177f
1 parent 34d9216
commit dee177f
Show file tree

Hide file tree

Showing 12 changed files with 621 additions and 605 deletions.
diff --git a/06_train/04_Train_Reviews_BERT_TensorFlow2_ScriptMode.ipynb b/06_train/04_Train_Reviews_BERT_TensorFlow2_ScriptMode.ipynb
diff --git a/06_train/src/requirements.txt b/06_train/src/requirements.txt
@@ -1,3 +1,11 @@
 # Add python dependencies here...
-scikit-learn==0.20.3
-nltk==3.4.5
+# SageMaker bug that requires us to do this in the code directly.
+#scikit-learn==0.20.3
+#tensorflow-hub==0.7.0
+#bert-tensorflow==1.0.2
+
+tensorflow==2.2.0-rc1
+grpcio
+tqdm
+bert-for-tf2
+sentencepiece
diff --git a/06_train/src/test-local.sh b/06_train/src/test-local.sh
@@ -1,4 +1,4 @@
-python copy_data_locally.py
-
-SM_CHANNEL_TRAIN=feature-store/amazon-reviews/csv/balanced-tfidf-without-header/train SM_CHANNEL_VALIDATION=feature-store/amazon-reviews/csv/balanced-tfidf-without-header/validation SM_MODEL_DIR=. python xgboost_reviews.py 
-# --num-rounds=10
+rm -rf model/
+rm -rf output/
+#SM_INPUT_DATA_CONFIG={\"train\":{\"TrainingInputMode\":\"Pipe\"}} 
+SM_CURRENT_HOST=blah SM_NUM_GPUS=0 SM_HOSTS={\"hosts\":\"blah\"} SM_MODEL_DIR=model/ SM_OUTPUT_DATA_DIR=output/ SM_CHANNEL_TRAIN=data/train SM_CHANNEL_VALIDATION=data/validation SM_CHANNEL_TEST=data/test python tf_bert_reviews.py --use-xla=False --use-amp=False --train-batch-size=8 --validation-batch-size=8 --test-batch-size=8 --epochs=2 --train-steps-per-epoch=10 --validation-steps=10 --test-steps=10 --max-seq-length=128 --freeze-bert-layer=False --enable-sagemaker-debugger=True
diff --git a/06_train/src/tf_bert_reviews.py b/06_train/src/tf_bert_reviews.py
diff --git a/06_train/src_bert_tf2/requirements.txt b/06_train/src_bert_tf2/requirements.txt
diff --git a/06_train/src_bert_tf2/test-local.sh b/06_train/src_bert_tf2/test-local.sh
diff --git a/06_train/src_bert_tf2/tf_bert_reviews.py b/06_train/src_bert_tf2/tf_bert_reviews.py
diff --git a/06_train/src/.gitignore → 06_train/wip/src/.gitignore b/06_train/src/.gitignore → 06_train/wip/src/.gitignore
diff --git a/06_train/src/copy_data_locally.py → 06_train/wip/src/copy_data_locally.py b/06_train/src/copy_data_locally.py → 06_train/wip/src/copy_data_locally.py
diff --git a/06_train/wip/src/requirements.txt b/06_train/wip/src/requirements.txt
@@ -0,0 +1,3 @@
+# Add python dependencies here...
+scikit-learn==0.20.3
+nltk==3.4.5
diff --git a/06_train/wip/src/test-local.sh b/06_train/wip/src/test-local.sh
@@ -0,0 +1,4 @@
+python copy_data_locally.py
+
+SM_CHANNEL_TRAIN=feature-store/amazon-reviews/csv/balanced-tfidf-without-header/train SM_CHANNEL_VALIDATION=feature-store/amazon-reviews/csv/balanced-tfidf-without-header/validation SM_MODEL_DIR=. python xgboost_reviews.py 
+# --num-rounds=10
diff --git a/06_train/src/xgboost_reviews.py → 06_train/wip/src/xgboost_reviews.py b/06_train/src/xgboost_reviews.py → 06_train/wip/src/xgboost_reviews.py