From a7ebd83ba057d222d776afc9bd4a3ad252d51020 Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:16:46 -0600 Subject: Create analysis-notebook-test-binder.yml Create a github action for binder tests based off [https://github.com/MobilityNet/mobilitynet-analysis-scripts/blob/master/.github/workflows/exploratory-notebooks-test-binder.yml](https://github.com/MobilityNet/mobilitynet-analysis-scripts/blob/master/.github/workflows/exploratory-notebooks-test-binder.yml) --- .../workflows/analysis-notebook-test-binder.yml | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 .github/workflows/analysis-notebook-test-binder.yml (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml new file mode 100644 index 0000000..9269866 --- /dev/null +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -0,0 +1,120 @@ +# This is a basic workflow to help you get started with Actions + +name: analysis-notebook-test-binder + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '5 4 * * 0' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # We would ideally have two jobs here called "build" and "test" + # However, in order to do that we would have to publish the image as an artifact + # to share it between jobs + # https://help.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts + # and it looks like the generated image is too large + # https://github.com/MobilityNet/mobilitynet-analysis-scripts/runs/613707818?check_suite_focus=true + # -rw-r--r-- 1 runner docker 3.3G Apr 23 21:37 /tmp/mobilitynet_84f67fcad71c.tar + # Filesystem Size Used Avail Use% Mounted on + # /dev/sda1 84G 79G 5.1G 94% / + # https://github.com/MobilityNet/mobilitynet-analysis-scripts/runs/612938777?check_suite_focus=true + # Fail to upload '/tmp/mobilitynet_ec3aba6d029f.tar' due to 'Blob is incomplete (missing block). + # while we figure out how to trim that, we have to run them as steps in the same job + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + outputs: + IMAGE_SHA_NAME: ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} + OUT_IMGNAME: ${{ steps.package-build.outputs.OUT_IMGNAME }} + strategy: + matrix: + # Two tests are sufficient because the other notebooks are already + # tested in the manual install and the timeline notebooks are almost + # identical + target: [ui_security_inventory_23_parsing.ipynb] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout + uses: actions/checkout@v2 + + - name: Create a docker image using repo2docker + id: install-using-repo2docker + uses: jupyterhub/repo2docker-action@master + with: + IMAGE_NAME: "toriis-analysis" + + - name: Verify repo2docker environment + run: | + echo "image SHA was ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }}" + echo "Listing all images" + docker image list + + echo "Checking to see if toriis-analysis exists" + docker image list | grep toriis-analysis | wc -l + echo "Checking to see if repo2docker creation image exists" + docker image list | grep repo2docker | wc -l + + echo "Reading versions of related software" + echo "---- DOCKER ----" + docker --version + CURR_DOCKER_VER=`docker --version` + echo "---- CONDA ----" + docker run ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} /srv/conda/bin/conda --version | cut -d " " -f 2 + CURR_CONDA_VER=`docker run ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} /srv/conda/bin/conda --version | cut -d " " -f 2` + echo "---- repo2docker----" + docker run jupyter/repo2docker:master pip3 list | grep jupyter-repo2docker + CURR_R2D_VER=`docker run jupyter/repo2docker:master pip3 list | grep jupyter-repo2docker` + echo "On checking, docker ver is $CURR_DOCKER_VER" + echo " conda ver is $CURR_CONDA_VER and" + echo " repo2docker ver is $CURR_R2D_VER" + + - name: Start the recently built docker container + run: | + echo "image SHA was ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }}" + docker run --name toriis-analysis_test -d -p 8888:8888 ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} jupyter notebook --ip 0.0.0.0 + + - name: Test the interactive timeline + run: | + echo "About to run notebooks" + docker exec toriis-analysis_test python bin/run_notebooks.py ${{ matrix.target }} + echo "After running the notebooks, checking container list" + docker container list + + - name: Copy the failure output + id: copy-failure-output + if: failure() + run: | + OUT_FILENAME=`echo ${{ matrix.target }} | sed "s/.ipynb/_out.ipynb/"` + echo "output filename = $OUT_FILENAME" + + echo "About to copy file from docker" + docker exec toriis-analysis_test ls + DOCKER_WD=`docker exec toriis-analysis_test pwd` + echo "DOCKER_WD=$DOCKER_WD" + docker exec toriis-analysis_test ls $DOCKER_WD + docker exec toriis-analysis_test ls $DOCKER_WD/$OUT_FILENAME + docker cp mobilitynet_test:$DOCKER_WD/$OUT_FILENAME /tmp + ls -al /tmp/$OUT_FILENAME + echo "::set-output name=OUT_FILENAME::${OUT_FILENAME}" + + - name: Upload result for the interactive timeline + if: failure() + uses: actions/upload-artifact@v1 + with: + name: ${{ steps.copy-failure-output.outputs.OUT_FILENAME }} + path: /tmp/${{ steps.copy-failure-output.outputs.OUT_FILENAME }} + + - name: Shutdown the recently built docker container + run: | + echo "Running docker container list" + docker container list + docker stop mobilitynet_test -- cgit v1.2.3 From 84e372d78cfe15f62f001e0ec7fa6409ce5d0c2e Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:19:41 -0600 Subject: Update analysis-notebook-test-binder.yml changed workflow to test on branch 2-set-up-binder-1 for now, need to change back if the test passes --- .github/workflows/analysis-notebook-test-binder.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml index 9269866..f47a715 100644 --- a/.github/workflows/analysis-notebook-test-binder.yml +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -6,9 +6,9 @@ name: analysis-notebook-test-binder # events but only for the master branch on: push: - branches: [ main ] + branches: [ 2-set-up-binder-1 ] pull_request: - branches: [ main ] + branches: [ 2-set-up-binder-1 ] schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' -- cgit v1.2.3 From 64cb5b512091d2c8cb956672b9706beeacc4797c Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:24:45 -0600 Subject: Update analysis-notebook-test-binder.yml switched branches back to main, will open pr and debug in there. --- .github/workflows/analysis-notebook-test-binder.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml index f47a715..b4e5469 100644 --- a/.github/workflows/analysis-notebook-test-binder.yml +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -3,30 +3,18 @@ name: analysis-notebook-test-binder # Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch +# events but only for the main branch on: push: - branches: [ 2-set-up-binder-1 ] + branches: [ main ] pull_request: - branches: [ 2-set-up-binder-1 ] + branches: [ main ] schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # We would ideally have two jobs here called "build" and "test" - # However, in order to do that we would have to publish the image as an artifact - # to share it between jobs - # https://help.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts - # and it looks like the generated image is too large - # https://github.com/MobilityNet/mobilitynet-analysis-scripts/runs/613707818?check_suite_focus=true - # -rw-r--r-- 1 runner docker 3.3G Apr 23 21:37 /tmp/mobilitynet_84f67fcad71c.tar - # Filesystem Size Used Avail Use% Mounted on - # /dev/sda1 84G 79G 5.1G 94% / - # https://github.com/MobilityNet/mobilitynet-analysis-scripts/runs/612938777?check_suite_focus=true - # Fail to upload '/tmp/mobilitynet_ec3aba6d029f.tar' due to 'Blob is incomplete (missing block). - # while we figure out how to trim that, we have to run them as steps in the same job build: # The type of runner that the job will run on runs-on: ubuntu-latest -- cgit v1.2.3 From cbd3822d77c19f3cfb5b4595be9fa62e11839728 Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:53:15 -0600 Subject: Update analysis-notebook-test-binder.yml change occurrences of mobilitynat to toriis-analysis --- .github/workflows/analysis-notebook-test-binder.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml index b4e5469..989fa65 100644 --- a/.github/workflows/analysis-notebook-test-binder.yml +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -90,7 +90,7 @@ jobs: echo "DOCKER_WD=$DOCKER_WD" docker exec toriis-analysis_test ls $DOCKER_WD docker exec toriis-analysis_test ls $DOCKER_WD/$OUT_FILENAME - docker cp mobilitynet_test:$DOCKER_WD/$OUT_FILENAME /tmp + docker cp toriis-analysis_test:$DOCKER_WD/$OUT_FILENAME /tmp ls -al /tmp/$OUT_FILENAME echo "::set-output name=OUT_FILENAME::${OUT_FILENAME}" @@ -105,4 +105,4 @@ jobs: run: | echo "Running docker container list" docker container list - docker stop mobilitynet_test + docker stop toriis-analysis -- cgit v1.2.3 From e747fe2a02de8277b626726c9e5afdc4acd6027a Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:05:21 -0600 Subject: Update analysis-notebook-test-binder.yml changed toriis-analysis to toriis, I believe the - character was causing issues. --- .../workflows/analysis-notebook-test-binder.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml index 989fa65..c2c2dc7 100644 --- a/.github/workflows/analysis-notebook-test-binder.yml +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -38,7 +38,7 @@ jobs: id: install-using-repo2docker uses: jupyterhub/repo2docker-action@master with: - IMAGE_NAME: "toriis-analysis" + IMAGE_NAME: "toriis" - name: Verify repo2docker environment run: | @@ -46,8 +46,8 @@ jobs: echo "Listing all images" docker image list - echo "Checking to see if toriis-analysis exists" - docker image list | grep toriis-analysis | wc -l + echo "Checking to see if toriis exists" + docker image list | grep toriis | wc -l echo "Checking to see if repo2docker creation image exists" docker image list | grep repo2docker | wc -l @@ -68,12 +68,12 @@ jobs: - name: Start the recently built docker container run: | echo "image SHA was ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }}" - docker run --name toriis-analysis_test -d -p 8888:8888 ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} jupyter notebook --ip 0.0.0.0 + docker run --name toriis_test -d -p 8888:8888 ${{ steps.install-using-repo2docker.outputs.IMAGE_SHA_NAME }} jupyter notebook --ip 0.0.0.0 - name: Test the interactive timeline run: | echo "About to run notebooks" - docker exec toriis-analysis_test python bin/run_notebooks.py ${{ matrix.target }} + docker exec toriis_test python bin/run_notebooks.py ${{ matrix.target }} echo "After running the notebooks, checking container list" docker container list @@ -85,12 +85,12 @@ jobs: echo "output filename = $OUT_FILENAME" echo "About to copy file from docker" - docker exec toriis-analysis_test ls - DOCKER_WD=`docker exec toriis-analysis_test pwd` + docker exec toriis_test ls + DOCKER_WD=`docker exec toriis_test pwd` echo "DOCKER_WD=$DOCKER_WD" - docker exec toriis-analysis_test ls $DOCKER_WD - docker exec toriis-analysis_test ls $DOCKER_WD/$OUT_FILENAME - docker cp toriis-analysis_test:$DOCKER_WD/$OUT_FILENAME /tmp + docker exec toriis_test ls $DOCKER_WD + docker exec toriis_test ls $DOCKER_WD/$OUT_FILENAME + docker cp toriis_test:$DOCKER_WD/$OUT_FILENAME /tmp ls -al /tmp/$OUT_FILENAME echo "::set-output name=OUT_FILENAME::${OUT_FILENAME}" @@ -105,4 +105,4 @@ jobs: run: | echo "Running docker container list" docker container list - docker stop toriis-analysis + docker stop toriis -- cgit v1.2.3 From 85b7be2354de05b877ba54d84b2c6845cf337902 Mon Sep 17 00:00:00 2001 From: Gabriel Kosmacher <73120774+kennykos@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:15:19 -0600 Subject: Update analysis-notebook-test-binder.yml --- .github/workflows/analysis-notebook-test-binder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to '.github/workflows') diff --git a/.github/workflows/analysis-notebook-test-binder.yml b/.github/workflows/analysis-notebook-test-binder.yml index c2c2dc7..e0185c0 100644 --- a/.github/workflows/analysis-notebook-test-binder.yml +++ b/.github/workflows/analysis-notebook-test-binder.yml @@ -36,7 +36,7 @@ jobs: - name: Create a docker image using repo2docker id: install-using-repo2docker - uses: jupyterhub/repo2docker-action@master + uses: shankari/repo2docker-action@master with: IMAGE_NAME: "toriis" -- cgit v1.2.3