diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index 6ceb1d060..1b139c978 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -3,9 +3,13 @@ name: release-docker on: push: branches: - - 'main' - 'develop' - 'docker' + tags: + - 'v*' + pull_request: + branches: + - 'develop' jobs: @@ -21,6 +25,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub + if: github.event_name != 'pull_request' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -40,9 +45,20 @@ jobs: uses: docker/build-push-action@v4 with: context: ./docker/ - push: true tags: ${{ steps.meta_dockerhub.outputs.tags }} labels: ${{ steps.meta_dockerhub.outputs.labels }} + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'push' }} + + - name: Update repo description + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: openml/openml-python + short-description: "pre-installed openml-python environment" + readme-filepath: ./docker/readme.md - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/docker/Dockerfile b/docker/Dockerfile index c27abba40..a84723309 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,8 @@ # Useful building docs or running unix tests from a Windows host. FROM python:3.10 -RUN git clone https://github.com/openml/openml-python.git omlp -WORKDIR omlp +RUN git clone https://github.com/openml/openml-python.git openml +WORKDIR openml RUN python -m venv venv RUN venv/bin/pip install wheel setuptools RUN venv/bin/pip install -e .[test,examples,docs,examples_unix] @@ -11,6 +11,8 @@ RUN venv/bin/pip install -e .[test,examples,docs,examples_unix] WORKDIR / RUN mkdir scripts ADD startup.sh scripts/ +ADD readme.md / + # Due to the nature of the Docker container it might often be built from Windows. # It is typical to have the files with \r\n line-ending, we want to remove it for the unix image. RUN sed -i 's/\r//g' scripts/startup.sh diff --git a/docker/readme.md b/docker/readme.md index 47ad6d23b..d0af9d9fe 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -1,86 +1,131 @@ # OpenML Python Container -This docker container has the latest development version of openml-python downloaded and pre-installed. -It can be used to run the unit tests or build the docs in a fresh and/or isolated unix environment. -Instructions only tested on a Windows host machine. +This docker container has the latest version of openml-python downloaded and pre-installed. +It can also be used by developers to run unit tests or build the docs in +a fresh and/or isolated unix environment. +This document contains information about: -First pull the docker image: + 1. [Usage](#usage): how to use the image and its main modes. + 2. [Using local or remote code](#using-local-or-remote-code): useful when testing your own latest changes. + 3. [Versions](#versions): identify which image to use. + 4. [Development](#for-developers): information about the Docker image for developers. - docker pull openml/openml-python +*note:* each docker image is shipped with a readme, which you can read with: +`docker run --entrypoint=/bin/cat openml/openml-python:TAG readme.md` ## Usage +There are three main ways to use the image: running a pre-installed Python environment, +running tests, and building documentation. - docker run -it openml/openml-python [DOC,TEST] [BRANCH] +### Running `Python` with pre-installed `OpenML-Python` (default): -The image is designed to work with two specified directories which may be mounted ([`docker --mount documentation`](https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount)). -You can mount your openml-python folder to the `/code` directory to run tests or build docs on your local files. -You can mount an `/output` directory to which the container will write output (currently only used for docs). -Each can be mounted by adding a `--mount type=bind,source=SOURCE,destination=/DESTINATION` where `SOURCE` is the absolute path to your code or output directory, and `DESTINATION` is either `code` or `output`. - -E.g. mounting a code directory: +To run `Python` with a pre-installed `OpenML-Python` environment run: - docker run -i --mount type=bind,source="E:\\repositories/openml-python",destination="/code" -t openml/openml-python +```text +docker run -it openml/openml-python +``` -E.g. mounting an output directory: +this accepts the normal `Python` arguments, e.g.: - docker run -i --mount type=bind,source="E:\\files/output",destination="/output" -t openml/openml-python +```text +docker run openml/openml-python -c "import openml; print(openml.__version__)" +``` -You can mount both at the same time. +if you want to run a local script, it needs to be mounted first. Mount it into the +`openml` folder: -### Bash (default) -By default bash is invoked, you should also use the `-i` flag when starting the container so it processes input: +``` +docker run -v PATH/TO/FILE:/openml/MY_SCRIPT.py openml/openml-python MY_SCRIPT.py +``` - docker run -it openml/openml-python +### Running unit tests -### Building Documentation -There are two ways to build documentation, either directly from the `HEAD` of a branch on Github or from your local directory. +You can run the unit tests by passing `test` as the first argument. +It also requires a local or remote repository to be specified, which is explained +[below]((#using-local-or-remote-code). For this example, we specify to test the +`develop` branch: -#### Building from a local repository -Building from a local directory requires you to mount it to the ``/code`` directory: +```text +docker run openml/openml-python test develop +``` - docker run --mount type=bind,source=PATH_TO_REPOSITORY,destination=/code -t openml/openml-python doc +### Building documentation -The produced documentation will be in your repository's ``doc/build`` folder. -If an `/output` folder is mounted, the documentation will *also* be copied there. +You can build the documentation by passing `doc` as the first argument, +you should [mount]((https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount)) +an output directory in which the docs will be stored. You also need to provide a remote +or local repository as explained in [the section below]((#using-local-or-remote-code). +In this example, we build documentation for the `develop` branch. +On Windows: -#### Building from an online repository -Building from a remote repository requires you to specify a branch. -The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/): +```text + docker run --mount type=bind,source="E:\\files/output",destination="/output" openml/openml-python doc develop +``` - docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output -t openml/openml-python doc BRANCH +on Linux: +```text + docker run --mount type=bind,source="./output",destination="/output" openml/openml-python doc develop +``` + +see [the section below]((#using-local-or-remote-code) for running against local changes +or a remote branch. -Where `BRANCH` is the name of the branch for which to generate the documentation. -It is also possible to build the documentation from the branch on a fork, in this case the `BRANCH` should be specified as `GITHUB_NAME#BRANCH` (e.g. `PGijsbers#my_feature`) and the name of the forked repository should be `openml-python`. +*Note: you can forgo mounting an output directory to test if the docs build successfully, +but the result will only be available within the docker container under `/openml/docs/build`.* -### Running tests -There are two ways to run tests, either directly from the `HEAD` of a branch on Github or from your local directory. -It works similar to building docs, but should specify `test` as mode. -For example, to run tests on your local repository: +## Using local or remote code - docker run --mount type=bind,source=PATH_TO_REPOSITORY,destination=/code -t openml/openml-python test - -Running tests from the state of an online repository is supported similar to building documentation (i.e. specify `BRANCH` instead of mounting `/code`). - -## Troubleshooting +You can build docs or run tests against your local repository or a Github repository. +In the examples below, change the `source` to match the location of your local repository. + +### Using a local repository + +To use a local directory, mount it in the `/code` directory, on Windows: + +```text + docker run --mount type=bind,source="E:\\repositories/openml-python",destination="/code" openml/openml-python test +``` -When you are mounting a directory you can check that it is mounted correctly by running the image in bash mode. -Navigate to the `/code` and `/output` directories and see if the expected files are there. -If e.g. there is no code in your mounted `/code`, you should double-check the provided path to your host directory. +on Linux: +```text + docker run --mount type=bind,source="/Users/pietergijsbers/repositories/openml-python",destination="/code" openml/openml-python test +``` -## Notes for developers -This section contains some notes about the structure of the image, intended for those who want to work on it. +when building docs, you also need to mount an output directory as shown above, so add both: + +```text +docker run --mount type=bind,source="./output",destination="/output" --mount type=bind,source="/Users/pietergijsbers/repositories/openml-python",destination="/code" openml/openml-python doc +``` + +### Using a Github repository +Building from a remote repository requires you to specify a branch. +The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/): + + docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output openml/openml-python [test,doc] BRANCH + +Where `BRANCH` is the name of the branch for which to generate the documentation. +It is also possible to build the documentation from the branch on a fork, +in this case the `BRANCH` should be specified as `GITHUB_NAME#BRANCH` (e.g. +`PGijsbers#my_feature_branch`) and the name of the forked repository should be `openml-python`. + +## For developers +This section contains some notes about the structure of the image, +intended for those who want to work on it. ### Added Directories The `openml/openml-python` image is built on a vanilla `python:3` image. -Additionally it contains the following files are directories: - - - `/omlp`: contains the openml-python repository in the state with which the image was built by default. - If working with a `BRANCH`, this repository will be set to the `HEAD` of `BRANCH`. - - `/omlp/venv/`: contains the used virtual environment for `doc` and `test`. It has `openml-python` dependencies pre-installed. - When invoked with `doc` or `test`, the dependencies will be updated based on the `setup.py` of the `BRANCH` or mounted `/code`. +Additionally, it contains the following files are directories: + + - `/openml`: contains the openml-python repository in the state with which the image + was built by default. If working with a `BRANCH`, this repository will be set to + the `HEAD` of `BRANCH`. + - `/openml/venv/`: contains the used virtual environment for `doc` and `test`. It has + `openml-python` dependencies pre-installed. When invoked with `doc` or `test`, the + dependencies will be updated based on the `setup.py` of the `BRANCH` or mounted `/code`. - `/scripts/startup.sh`: the entrypoint of the image. Takes care of the automated features (e.g. `doc` and `test`). ## Building the image -To build the image yourself, execute `docker build -f Dockerfile .` from this directory. -It will use the `startup.sh` as is, so any local changes will be present in the image. +To build the image yourself, execute `docker build -f Dockerfile .` from the `docker` +directory of the `openml-python` repository. It will use the `startup.sh` as is, so any +local changes will be present in the image. diff --git a/docker/startup.sh b/docker/startup.sh index 2a75a621c..34a5c61f3 100644 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -1,3 +1,6 @@ +# Entry script to switch between the different Docker functionalities. +# By default, execute Python with OpenML pre-installed +# # Entry script to allow docker to be ran for bash, tests and docs. # The script assumes a code repository can be mounted to ``/code`` and an output directory to ``/output``. # Executes ``mode`` on ``branch`` or the provided ``code`` directory. @@ -10,10 +13,11 @@ # Can be a branch on a Github fork, specified with the USERNAME#BRANCH format. # The test or doc build is executed on this branch. -if [ -z "$1" ]; then - echo "Executing in BASH mode." - bash - exit +if [[ ! ( $1 = "doc" || $1 = "test" ) ]]; then + cd openml + source venv/bin/activate + python "$@" + exit 0 fi # doc and test modes require mounted directories and/or specified branches @@ -32,8 +36,8 @@ if [ "$1" == "doc" ] && [ -n "$2" ] && ! [ -d "/output" ]; then fi if [ -n "$2" ]; then - # if a branch is provided, we will pull it into the `omlp` local repository that was created with the image. - cd omlp + # if a branch is provided, we will pull it into the `openml` local repository that was created with the image. + cd openml if [[ $2 == *#* ]]; then # If a branch is specified on a fork (with NAME#BRANCH format), we have to construct the url before pulling # We add a trailing '#' delimiter so the second element doesn't get the trailing newline from <<< @@ -52,12 +56,12 @@ if [ -n "$2" ]; then exit 1 fi git pull - code_dir="/omlp" + code_dir="/openml" else code_dir="/code" fi -source /omlp/venv/bin/activate +source /openml/venv/bin/activate cd $code_dir # The most recent ``main`` is already installed, but we want to update any outdated dependencies pip install -e .[test,examples,docs,examples_unix] @@ -71,6 +75,6 @@ if [ "$1" == "doc" ]; then make html make linkcheck if [ -d "/output" ]; then - cp -r /omlp/doc/build /output + cp -r /openml/doc/build /output fi -fi +fi \ No newline at end of file